diff --git a/alembic/pfsa-db01-gb/alembic/versions/20241220-145409_0b72e62efd44_add_partner_column_in_the_proposal_.py b/alembic/pfsa-db01-gb/alembic/versions/20241220-145409_0b72e62efd44_add_partner_column_in_the_proposal_.py new file mode 100644 index 0000000..862fb74 --- /dev/null +++ b/alembic/pfsa-db01-gb/alembic/versions/20241220-145409_0b72e62efd44_add_partner_column_in_the_proposal_.py @@ -0,0 +1,41 @@ +"""add partner column in the proposal table and partner table + +Revision ID: 0b72e62efd44 +Revises: b68482e38606 +Create Date: 2024-12-20 14:54:09.851011 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '0b72e62efd44' +down_revision = 'b68482e38606' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('partner', + sa.Column('partner_id', sa.Integer(), autoincrement=False, nullable=False, comment='Unique identifier of the partner'), + sa.Column('partner_name', sa.String(), nullable=False, comment='Name of the partner (e.g., subaru, gemini, keck, and uh)'), + sa.Column('partner_description', sa.String(), nullable=True, comment='Description of the partner'), + sa.Column('created_at', sa.DateTime(), server_default=sa.text("TIMEZONE('utc', CURRENT_TIMESTAMP)"), nullable=True, comment='The date and time in UTC when the record was created'), + sa.Column('updated_at', sa.DateTime(), nullable=True, comment='The date and time in UTC when the record was last updated'), + sa.PrimaryKeyConstraint('partner_id'), + sa.UniqueConstraint('partner_id'), + sa.UniqueConstraint('partner_name') + ) + op.add_column('proposal', sa.Column('partner_id', sa.Integer(), nullable=True)) + op.create_foreign_key(None, 'proposal', 'partner', ['partner_id'], ['partner_id']) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(None, 'proposal', type_='foreignkey') + op.drop_column('proposal', 'partner_id') + op.drop_table('partner') + # ### end Alembic commands ### diff --git a/diagrams/erdiagram_targetdb-20241220155829.pdf b/diagrams/erdiagram_targetdb-20241220155829.pdf new file mode 100644 index 0000000..4e38b84 Binary files /dev/null and b/diagrams/erdiagram_targetdb-20241220155829.pdf differ diff --git a/docs/examples/obsproc.md b/docs/examples/obsproc.md index 9aeb799..186c87d 100644 --- a/docs/examples/obsproc.md +++ b/docs/examples/obsproc.md @@ -68,9 +68,14 @@ z_sdss,SDSS z filter ```csv title="proposal_categories.csv" proposal_category_id,proposal_category_name,proposal_category_description 1,openuse,Subaru openuse proposal -2,keck,Subaru/Keck time exchange proposal -3,gemini,Subaru/Gemini time exchange proposal -4,uh,University of Hawaii proposal +``` + +```csv title="partner.csv" +partner_id,partner_name,partner_description +1,subaru,Subaru Telescope +2,keck,"W. M. Keck Observatory" +3,gemini,Gemini Telescope +4,uh,University of Hawaii ``` ```csv title="pfs_arm.csv" @@ -100,6 +105,7 @@ You can insert these data into the database using the following commands: ```console $ pfs-targetdb-cli insert filter_names.csv -c db_config.toml --table filter_name --commit $ pfs-targetdb-cli insert proposal_categories.csv -c db_config.toml --table proposal_category --commit +$ pfs-targetdb-cli insert partner.csv -c db_config.toml --table partner --commit $ pfs-targetdb-cli insert pfs_arm.csv -c db_config.toml --table pfs_arm --commit $ pfs-targetdb-cli insert target_types.csv -c db_config.toml --table target_type --commit ``` @@ -169,7 +175,7 @@ Suppose you have an Excel file named `pfs_allocation_summary.xlsx` with 2 sheets **`Proposals` Sheet**: | proposal_id | input_catalog_name | input_catalog_description | group_id | pi_first_name | pi_last_name | pi_middle_name | proposal_category_name | upload_id | n_obj | fiberhour_total | fiberhour_lr | fiberhour_mr | rot_total | rot_lr | rot_mr | -|-------------|--------------------|---------------------------|----------|---------------|--------------|----------------|------------------------|------------------|-------|-----------------|--------------|--------------|-----------|--------|--------| +| ----------- | ------------------ | ------------------------- | -------- | ------------- | ------------ | -------------- | ---------------------- | ---------------- | ----- | --------------- | ------------ | ------------ | --------- | ------ | ------ | | S99A-QT001 | pfs_example_1 | Example target list 1 | o99101 | Eiichi | Shibusawa | | openuse | d6e94eae259faf4e | 1572 | 379.5 | 379.5 | | 5.2 | 5.2 | | | S99A-QT002 | pfs_example_2 | Example target list 2 | o99102 | Umeko | Tsuda | | openuse | 5f695375c60f34c7 | 9712 | 17504 | 17504 | | 15.83 | 15.83 | | | S99A-QT003 | pfs_example_3 | Example target list 3 | o99103 | Shibasaburo | Kitasato | | openuse | ba59115da8084653 | 2047 | 395.25 | 395.25 | | 12.7 | 12.7 | | @@ -177,7 +183,7 @@ Suppose you have an Excel file named `pfs_allocation_summary.xlsx` with 2 sheets **`Allocation` Sheet**: | proposal_id | grade | rank | allocated_rot_total | allocated_rot_lr | allocated_rot_mr | allocated_time_total | allocated_time_lr | allocated_time_mr | n_ppc | allocation_rate_lr | allocation_rate_mr | completion_rate_lr | completion_rate_mr | -|-------------|-------|------|---------------------|------------------|------------------|----------------------|-------------------|-------------------|-------|--------------------|--------------------|--------------------|--------------------| +| ----------- | ----- | ---- | ------------------- | ---------------- | ---------------- | -------------------- | ----------------- | ----------------- | ----- | ------------------ | ------------------ | ------------------ | ------------------ | | S99A-QT001 | A | 9 | 2.8 | | 2.8 | 284.25 | 0 | 284.25 | 9 | 0.749011858 | | 0.723 | | | S99A-QT002 | B | 6.5 | 6.5 | 6.5 | | 8140.5 | 8140.5 | 0 | 21 | 0.465065128 | | 0.279 | | | S99A-QT003 | B | 6 | 9.6 | 9.6 | | 350.25 | 350.25 | 0 | 31 | 0.886148008 | | 0.684 | | @@ -268,4 +274,4 @@ pfs-targetdb-cli insert -c dbconf.toml -t user_pointing pointing_list.ecsv \ --commit --upload_id "aabbccddeeffgghh" ``` -Currently, you need to insert the custom pointing list one by one. We are planning to support batch insertion in the future. \ No newline at end of file +Currently, you need to insert the custom pointing list one by one. We are planning to support batch insertion in the future. diff --git a/docs/schema/partner.md b/docs/schema/partner.md new file mode 100644 index 0000000..45bc29d --- /dev/null +++ b/docs/schema/partner.md @@ -0,0 +1,32 @@ +# `partner` + +## Overview + +The `partner` table is used to store the information on partner institutions. + +## Columns + +Here are the columns in the `partner` table: + +| Column Name | Type | Description | Required[^1] | Default | +| ------------------- | -------- | ---------------------------------------------------------- | ------------ | ------- | +| partner_id | int | The unique identifier of the partner | \* | | +| partner_name | str | The name of the partner | \* | | +| partner_description | str | A brief description of the partner | | "" | +| created_at | datetime | The date and time in UTC when the record was created. | | | +| updated_at | datetime | The date and time in UTC when the record was last updated. | | | + +[^1]: Required when inserted by using the [CLI tool](../reference/cli.md) or equivalent functions. + +## Unique constraint + +- `partner_id` is the primary key of the table and must be unique. However, it not set as an auto-increment column. + +## Available proposal categories + +| partner_id | partner_name | partner_description | +| ---------: | ------------ | ---------------------- | +| 1 | subaru | Subaru Telescope | +| 2 | keck | W. M. Keck Observatory | +| 3 | gemini | Gemini Observatory | +| 4 | uh | University of Hawaii | diff --git a/docs/schema/proposal.md b/docs/schema/proposal.md index 2d6d211..e73e74d 100644 --- a/docs/schema/proposal.md +++ b/docs/schema/proposal.md @@ -9,7 +9,7 @@ The `proposal` table is used to store the information on proposals. Here are the columns in the `proposal` table: | Column Name | Type | Description | Unit | Required[^1] | Default | -|----------------------|----------|---------------------------------------------------------------------|------|--------------|---------| +| -------------------- | -------- | ------------------------------------------------------------------- | ---- | ------------ | ------- | | proposal_id | str | Proposal-ID (e.g, S23B-QN901) | | \* | | | group_id | str | Group ID issued by STARS (e.g., o21195) | | \* | | | pi_first_name | str | PI's first name | | | "" | @@ -21,6 +21,7 @@ Here are the columns in the `proposal` table: | allocated_time_lr | float | Allocated fiberhours for low-resolution mode | h | | 0 | | allocated_time_mr | float | Allocated fiberhours for medium-resolution mode | h | | 0 | | proposal_category_id | int | `proposal_category_id` in the `proposal_category` table | | | 0 | +| partner_id | int | `partner_id` in the `partner` table | | | | | is_too | bool | A flag to indicate if the proposal is a Target of Opportunity (ToO) | | | False | | created_at | datetime | The date and time in UTC when the record was created. | | | | | updated_at | datetime | The date and time in UTC when the record was last updated. | | | | @@ -35,6 +36,7 @@ Here are the columns in the `proposal` table: ## Foreign key constraints - `proposal_category_id` is a foreign key constraint that references the `proposal_category_id` in the `proposal_category` table. +- `partner_id` is a foreign key constraint that references the `partner_id` in the `partner` table ## Notes diff --git a/docs/schema/proposal_category.md b/docs/schema/proposal_category.md index a14eaac..b3406d6 100644 --- a/docs/schema/proposal_category.md +++ b/docs/schema/proposal_category.md @@ -24,9 +24,6 @@ Here are the columns in the `proposal_category` table: ## Available proposal categories -| proposal_category_id | proposal_category_name | proposal_category_description | -| -------------------: | ---------------------- | ------------------------------------ | -| 1 | openuse | Subaru openuse proposal | -| 2 | keck | Subaru/Keck time exchange proposal | -| 3 | gemini | Subaru/Gemini time exchange proposal | -| 4 | uh | University of Hawaii proposal | +| proposal_category_id | proposal_category_name | proposal_category_description | +| -------------------: | ---------------------- | ----------------------------- | +| 1 | openuse | Subaru openuse proposal | diff --git a/docs/tbls/README.md b/docs/tbls/README.md index 52450d8..b15a707 100644 --- a/docs/tbls/README.md +++ b/docs/tbls/README.md @@ -15,6 +15,7 @@ | [public.fluxstd](public.fluxstd.md) | 61 | | BASE TABLE | | [public.target](public.target.md) | 61 | | BASE TABLE | | [public.cluster](public.cluster.md) | 10 | | BASE TABLE | +| [public.partner](public.partner.md) | 5 | | BASE TABLE | ## Stored procedures and functions @@ -310,6 +311,13 @@ erDiagram timestamp_without_time_zone created_at timestamp_without_time_zone updated_at } +"public.partner" { + integer partner_id + varchar partner_name + varchar partner_description + timestamp_without_time_zone created_at + timestamp_without_time_zone updated_at +} ``` --- diff --git a/docs/tbls/public.partner.md b/docs/tbls/public.partner.md new file mode 100644 index 0000000..e29cc6a --- /dev/null +++ b/docs/tbls/public.partner.md @@ -0,0 +1,46 @@ +# public.partner + +## Description + +## Columns + +| Name | Type | Default | Nullable | Children | Parents | Comment | +| ---- | ---- | ------- | -------- | -------- | ------- | ------- | +| partner_id | integer | | false | | | Unique identifier of the partner | +| partner_name | varchar | | false | | | Name of the partner (e.g., subaru, gemini, keck, and uh) | +| partner_description | varchar | | true | | | Description of the partner | +| created_at | timestamp without time zone | timezone('utc'::text, CURRENT_TIMESTAMP) | true | | | The date and time in UTC when the record was created | +| updated_at | timestamp without time zone | | true | | | The date and time in UTC when the record was last updated | + +## Constraints + +| Name | Type | Definition | +| ---- | ---- | ---------- | +| partner_pkey | PRIMARY KEY | PRIMARY KEY (partner_id) | +| partner_partner_name_key | UNIQUE | UNIQUE (partner_name) | + +## Indexes + +| Name | Definition | +| ---- | ---------- | +| partner_pkey | CREATE UNIQUE INDEX partner_pkey ON public.partner USING btree (partner_id) | +| partner_partner_name_key | CREATE UNIQUE INDEX partner_partner_name_key ON public.partner USING btree (partner_name) | + +## Relations + +```mermaid +erDiagram + + +"public.partner" { + integer partner_id + varchar partner_name + varchar partner_description + timestamp_without_time_zone created_at + timestamp_without_time_zone updated_at +} +``` + +--- + +> Generated by [tbls](https://github.com/k1LoW/tbls) diff --git a/examples/data/partner.csv b/examples/data/partner.csv new file mode 100644 index 0000000..fd97bf1 --- /dev/null +++ b/examples/data/partner.csv @@ -0,0 +1,5 @@ +partner_id,partner_name,partner_description +1,subaru,Subaru Telescope +2,keck,"W. M. Keck Observatory" +3,gemini,Gemini Telescope +4,uh,University of Hawaii diff --git a/examples/data/proposal_category.csv b/examples/data/proposal_category.csv index 14c3b8e..c29956a 100644 --- a/examples/data/proposal_category.csv +++ b/examples/data/proposal_category.csv @@ -1,5 +1,2 @@ proposal_category_id,proposal_category_name,proposal_category_description 1,openuse,Subaru openuse proposal -2,keck,Subaru/Keck time exchange proposal -3,gemini,Subaru/Gemini time exchange proposal -4,uh,University of Hawaii proposal \ No newline at end of file diff --git a/examples/docker/Dockerfile b/examples/docker/Dockerfile index 0698e41..58f1310 100644 --- a/examples/docker/Dockerfile +++ b/examples/docker/Dockerfile @@ -1,6 +1,7 @@ # Use the official Docker Hub Postgres 10.6 image # FROM postgres:10.6 -FROM postgres:12 +# FROM postgres:12 +FROM postgres:16 # Set environment variables ENV POSTGRES_USER "" @@ -11,8 +12,11 @@ ENV POSTGRES_DB "" RUN apt-get update && apt-get install -y \ git \ build-essential \ + liblz4-dev \ + libreadline-dev \ + libzstd-dev \ zlib1g-dev \ - postgresql-server-dev-12 + postgresql-server-dev-16 # Clone the q3c extension from GitHub RUN git clone https://github.com/segasai/q3c.git @@ -31,4 +35,4 @@ EXPOSE 5432 VOLUME ["/etc/postgresql", "/var/log/postgresql", "/var/lib/postgresql"] # Set the default command to run when starting the container -CMD ["postgres"] \ No newline at end of file +CMD ["postgres"] diff --git a/src/targetdb/cli/cli_main.py b/src/targetdb/cli/cli_main.py index 26bf315..833c09c 100644 --- a/src/targetdb/cli/cli_main.py +++ b/src/targetdb/cli/cli_main.py @@ -47,6 +47,7 @@ class TargetdbTable(str, Enum): filter_name = "filter_name" fluxstd = "fluxstd" input_catalog = "input_catalog" + partner = "partner" pfs_arm = "pfs_arm" proposal = "proposal" proposal_category = "proposal_category" diff --git a/src/targetdb/models/__init__.py b/src/targetdb/models/__init__.py index 6d3e9b4..a5d9d0d 100644 --- a/src/targetdb/models/__init__.py +++ b/src/targetdb/models/__init__.py @@ -36,6 +36,7 @@ def pg_utcnow(element, compiler, **kw): from .filter_name import filter_name # noqa E402 from .pfs_arm import pfs_arm # noqa E402 from .proposal_category import proposal_category # noqa E402 +from .partner import partner # noqa E402 # from .proposal_grade import proposal_grade # noqa E402 from .input_catalog import input_catalog # noqa E402 @@ -54,6 +55,7 @@ def pg_utcnow(element, compiler, **kw): "filter_name", "pfs_arm", "proposal_category", + "partner", # "proposal_grade", "input_catalog", "user_pointing", diff --git a/src/targetdb/models/partner.py b/src/targetdb/models/partner.py new file mode 100644 index 0000000..eaa47ec --- /dev/null +++ b/src/targetdb/models/partner.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python + +from sqlalchemy import ( + Column, + DateTime, + Integer, + String, +) +from sqlalchemy.orm import relationship + +from . import Base, comment_created_at, comment_updated_at, utcnow + + +class partner(Base): + __tablename__ = "partner" + + partner_id = Column( + Integer, + primary_key=True, + unique=True, + autoincrement=False, + comment="Unique identifier of the partner", + ) + partner_name = Column( + String, + unique=True, + nullable=False, + comment="Name of the partner (e.g., subaru, gemini, keck, and uh)", + ) + partner_description = Column( + String, default="", comment="Description of the partner" + ) + created_at = Column( + DateTime, + comment=comment_created_at, + server_default=utcnow(), + ) + updated_at = Column( + DateTime, + comment=comment_updated_at, + onupdate=utcnow(), + ) + + proposals = relationship("proposal", backref="partner") diff --git a/src/targetdb/models/proposal.py b/src/targetdb/models/proposal.py index 5a48a53..ff45d8b 100644 --- a/src/targetdb/models/proposal.py +++ b/src/targetdb/models/proposal.py @@ -7,6 +7,7 @@ Base, comment_created_at, comment_updated_at, + partner, proposal_category, # proposal_grade, utcnow, @@ -61,29 +62,15 @@ class proposal(Base): Integer, ForeignKey("proposal_category.proposal_category_id") ) - # is_queue = Column( - # Boolean, - # default=True, - # comment="True when the proposal contains queue-mode allocation", - # ) - # is_classical = Column( - # Boolean, - # default=False, - # comment="True when the proposal contains classical-mode allocation", - # ) + partner_id = Column( + Integer, + ForeignKey("partner.partner_id"), + ) is_too = Column(Boolean, default=False, comment="True when the proposal is ToO") - created_at = Column( - DateTime, - comment=comment_created_at, - server_default=utcnow(), - ) - updated_at = Column( - DateTime, - comment=comment_updated_at, - onupdate=utcnow(), - ) + created_at = Column(DateTime, comment=comment_created_at, server_default=utcnow()) + updated_at = Column(DateTime, comment=comment_updated_at, onupdate=utcnow()) proposal_categories = relationship(proposal_category, backref=backref("proposal")) # proposal_grades = relationship("proposal_grade", back_populates="proposal") @@ -101,6 +88,7 @@ def __init__( allocated_time_lr, allocated_time_mr, proposal_category_id, + partner_id, is_too, created_at, updated_at, @@ -116,6 +104,7 @@ def __init__( self.allocated_time_lr = allocated_time_lr self.allocated_time_mr = allocated_time_mr self.proposal_category_id = proposal_category_id + self.partner_id = partner_id self.is_too = is_too self.created_at = created_at self.updated_at = updated_at diff --git a/src/targetdb/utils.py b/src/targetdb/utils.py index 3a9d364..ad8c2c1 100644 --- a/src/targetdb/utils.py +++ b/src/targetdb/utils.py @@ -307,7 +307,8 @@ def draw_diagram( outfile = os.path.join(output_dir, f"{sc_outprefix}-{time_string}.pdf") comm = [ - f"{os.path.join(config['schemacrawler']['SCHEMACRAWLERDIR'],'_schemacrawler/bin/schemacrawler.sh')}", + f"{os.path.join(config['schemacrawler']['SCHEMACRAWLERDIR'],'bin/schemacrawler.sh')}", + # f"{os.path.join(config['schemacrawler']['SCHEMACRAWLERDIR'],'_schemacrawler/bin/schemacrawler.sh')}", "--command=schema", "--server=postgresql", f"--host={config['targetdb']['db']['host']}",