Merge pull request #103 from Subaru-PFS/tickets/OBSPROC-101

Add `user_pointing` table to store custom pointing centers

alembic/pfsa-db01-gb/alembic/versions/20241024-120338_b68482e38606_add_user_pointing_table.py

@@ -0,0 +1,109 @@
+"""add user_pointing table
+
+Revision ID: b68482e38606
+Revises: f07c300ae87f
+Create Date: 2024-10-24 12:03:38.150098
+
+"""
+
+import sqlalchemy as sa
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "b68482e38606"
+down_revision = "f07c300ae87f"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "user_pointing",
+        sa.Column(
+            "user_pointing_id",
+            sa.BigInteger(),
+            nullable=False,
+            comment="Unique identifier for each user-defined pointing (autoincremented primary key)",
+        ),
+        sa.Column(
+            "ppc_code",
+            sa.String(),
+            nullable=False,
+            comment="String identifier of the pointing set either by the uploader or user",
+        ),
+        sa.Column(
+            "ppc_ra",
+            sa.Float(),
+            nullable=False,
+            comment="RA of the pointing center (ICRS, degree)",
+        ),
+        sa.Column(
+            "ppc_dec",
+            sa.Float(),
+            nullable=False,
+            comment="Dec of the pointing center (ICRS, degree)",
+        ),
+        sa.Column(
+            "ppc_pa",
+            sa.Float(),
+            nullable=False,
+            comment="Position angle of the pointing center (degree)",
+        ),
+        sa.Column(
+            "ppc_resolution",
+            sa.Enum("L", "M", name="resolutionmode"),
+            nullable=False,
+            comment="Resolution mode of the pointing ('L' or 'M')",
+        ),
+        sa.Column(
+            "ppc_priority",
+            sa.Float(),
+            nullable=False,
+            comment="Priority of the pointing calculated by the uploader",
+        ),
+        sa.Column(
+            "input_catalog_id",
+            sa.Integer(),
+            nullable=False,
+            comment="Input catalog ID from the input_catalog table",
+        ),
+        sa.Column(
+            "created_at",
+            sa.DateTime(),
+            server_default=sa.text("TIMEZONE('utc', CURRENT_TIMESTAMP)"),
+            nullable=True,
+            comment="The date and time in UTC when the record was created",
+        ),
+        sa.Column(
+            "updated_at",
+            sa.DateTime(),
+            nullable=True,
+            comment="The date and time in UTC when the record was last updated",
+        ),
+        sa.ForeignKeyConstraint(
+            ["input_catalog_id"],
+            ["input_catalog.input_catalog_id"],
+        ),
+        sa.PrimaryKeyConstraint("user_pointing_id"),
+    )
+    op.add_column(
+        "input_catalog",
+        sa.Column(
+            "is_user_pointing",
+            sa.Boolean(),
+            nullable=True,
+            comment="True if user-defined pointings are provided",
+        ),
+    )
+    # op.drop_index('target_q3c_ang2ipix_idx1', table_name='target')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    # op.create_index('target_q3c_ang2ipix_idx1', 'target', [sa.text('q3c_ang2ipix(ra, "dec")')], unique=False)
+    op.drop_column("input_catalog", "is_user_pointing")
+    op.drop_table("user_pointing")
+    # ### end Alembic commands ###

docs/examples/obsproc.md

@@ -169,15 +169,15 @@ Suppose you have an Excel file named `pfs_allocation_summary.xlsx` with 2 sheets
 **`Proposals` Sheet**:
 
 | proposal_id | input_catalog_name | input_catalog_description | group_id | pi_first_name | pi_last_name | pi_middle_name | proposal_category_name | upload_id        | n_obj | fiberhour_total | fiberhour_lr | fiberhour_mr | rot_total | rot_lr | rot_mr |
-| ----------- | ------------------ | ------------------------- | -------- | ------------- | ------------ | -------------- | ---------------------- | ---------------- | ----- | --------------- | ------------ | ------------ | --------- | ------ | ------ |
+|-------------|--------------------|---------------------------|----------|---------------|--------------|----------------|------------------------|------------------|-------|-----------------|--------------|--------------|-----------|--------|--------|
 | S99A-QT001  | pfs_example_1      | Example target list 1     | o99101   | Eiichi        | Shibusawa    |                | openuse                | d6e94eae259faf4e | 1572  | 379.5           | 379.5        |              | 5.2       | 5.2    |        |
 | S99A-QT002  | pfs_example_2      | Example target list 2     | o99102   | Umeko         | Tsuda        |                | openuse                | 5f695375c60f34c7 | 9712  | 17504           | 17504        |              | 15.83     | 15.83  |        |
 | S99A-QT003  | pfs_example_3      | Example target list 3     | o99103   | Shibasaburo   | Kitasato     |                | openuse                | ba59115da8084653 | 2047  | 395.25          | 395.25       |              | 12.7      | 12.7   |        |
 
 **`Allocation` Sheet**:
 
 | proposal_id | grade | rank | allocated_rot_total | allocated_rot_lr | allocated_rot_mr | allocated_time_total | allocated_time_lr | allocated_time_mr | n_ppc | allocation_rate_lr | allocation_rate_mr | completion_rate_lr | completion_rate_mr |
-| ----------- | ----- | ---- | ------------------- | ---------------- | ---------------- | -------------------- | ----------------- | ----------------- | ----- | ------------------ | ------------------ | ------------------ | ------------------ |
+|-------------|-------|------|---------------------|------------------|------------------|----------------------|-------------------|-------------------|-------|--------------------|--------------------|--------------------|--------------------|
 | S99A-QT001  | A     | 9    | 2.8                 |                  | 2.8              | 284.25               | 0                 | 284.25            | 9     | 0.749011858        |                    | 0.723              |                    |
 | S99A-QT002  | B     | 6.5  | 6.5                 | 6.5              |                  | 8140.5               | 8140.5            | 0                 | 21    | 0.465065128        |                    | 0.279              |                    |
 | S99A-QT003  | B     | 6    | 9.6                 | 9.6              |                  | 350.25               | 350.25            | 0                 | 31    | 0.886148008        |                    | 0.684              |                    |
@@ -198,10 +198,10 @@ S99A-QT003,o99103,Shibasaburo,Kitasato,,6.0,B,350.25,350.25,0.0,openuse,true
 ```
 
 ```csv title="input_catalogs.csv"
-input_catalog_name,input_catalog_description,upload_id,proposal_id,is_classical
-pfs_example_1,Example target list 1,d6e94eae259faf4e,S99A-QT001,false
-pfs_example_2,Example target list 2,5f695375c60f34c7,S99A-QT002,false
-pfs_example_3,Example target list 3,ba59115da8084653,S99A-QT003,true
+input_catalog_name,input_catalog_description,upload_id,proposal_id,is_classical,is_user_pointing
+pfs_example_1,Example target list 1,d6e94eae259faf4e,S99A-QT001,false,false
+pfs_example_2,Example target list 2,5f695375c60f34c7,S99A-QT002,false,false
+pfs_example_3,Example target list 3,ba59115da8084653,S99A-QT003,true,true
 ```
 
 ### Insert to the `proposal` and `input_catalog` tables for `target` data
@@ -258,3 +258,14 @@ $ pfs-targetdb-cli insert 20240221-001431-d6e94eae259faf4e/target_d6e94eae259faf
 $ pfs-targetdb-cli insert 20240228-050253-5f695375c60f34c7/target_5f695375c60f34c7.ecsv -c db_config.toml -t target --from-uploader --upload_id 5f695375c60f34c7 --proposal_id S99A-QT002 --commit
 $ pfs-targetdb-cli insert 20240229-013729-ba59115da8084653/target_ba59115da8084653.ecsv -c db_config.toml -t target --from-uploader --upload_id ba59115da8084653 --proposal_id S99A-QT003 --commit
 ```
+
+## Insert user-defined pointing lists into the user_pointing table
+
+When there is a user-defined pointing list, it can be inserted into the `user_pointing` table in the `targetdb` database.
+
+```bash
+pfs-targetdb-cli insert -c dbconf.toml -t user_pointing pointing_list.ecsv \
+    --commit --upload_id "aabbccddeeffgghh"
+```
+
+Currently, you need to insert the custom pointing list one by one. We are planning to support batch insertion in the future.

docs/schema/input_catalog.md

@@ -9,13 +9,14 @@ The `input_catalog` table is used to store the information on catalogs for sourc
 Here are the columns in the `input_catalog` table:
 
 | Column Name               | Type     | Description                                                           | Required[^1] | Default |
-|---------------------------|----------|-----------------------------------------------------------------------|--------------|---------|
+| ------------------------- | -------- | --------------------------------------------------------------------- | ------------ | ------- |
 | input_catalog_id          | int      | The ID of the input catalog                                           | (\*)         |         |
 | input_catalog_name        | str      | The name of the input catalog                                         | \*           |         |
 | input_catalog_description | str      | A brief description of the input catalog                              |              | ""      |
 | upload_id                 | str      | A 16-character string assigned at the submission of the target list   |              | ""      |
 | active                    | bool     | A flag to indicate if the catalog is active                           |              | True    |
 | is_classical              | bool     | A flag to indicate if the catalog is for a classical-mode observation |              | False   |
+| is_user_pointing          | bool     | A flag to indicate if user-defined pointings are provided             |              | False   |
 | created_at                | datetime | The date and time in UTC when the record was created.                 |              |         |
 | updated_at                | datetime | The date and time in UTC when the record was last updated.            |              |         |
 
@@ -26,7 +27,7 @@ Here are the columns in the `input_catalog` table:
 - `input_catalog_id` is a unique constraint and the primary key of the table. The `input_catalog_id` column is used as follows.
 
 | Start |   End | Description                                           |
-|------:|------:|-------------------------------------------------------|
+| ----: | ----: | ----------------------------------------------------- |
 |     0 |   999 | Reserved for major survey catalogs                    |
 |  1000 |  2999 | Sky object catalogs                                   |
 |  3000 |  3999 | Flux standard star catalogs                           |
@@ -43,7 +44,7 @@ Command line tools will check the IDs when one tries to insert a new record with
 These are example of the available input catalogs.
 
 | input_catalog_id | input_catalog_name   | input_catalog_description                      |
-|-----------------:|----------------------|------------------------------------------------|
+| ---------------: | -------------------- | ---------------------------------------------- |
 |                0 | simulated            | simulated catalog                              |
 |                1 | gaia_dr1             | Gaia Data Release 1                            |
 |                2 | gaia_dr2             | Gaia Data Release 2                            |

docs/schema/user_pointing.md

@@ -0,0 +1,32 @@
+# `user_pointing`
+
+## Overview
+
+The `user_pointing` table is used to store the information on user-defined pointings.
+
+## Columns
+
+Here are the columns in the `user_pointing` table:
+
+| Column Name      | Type     | Description                                                | Unit | Required[^1] | Default |
+| ---------------- | -------- | ---------------------------------------------------------- | ---- | ------------ | ------- |
+| user_pointing_id | int64    | The unique identifier of the record                        |      |              |         |
+| ppc_code         | str      | Name of the pointing                                       |      | \*           |         |
+| ppc_ra           | float    | RA at the pointing center                                  | deg  | \*           |         |
+| ppc_dec          | float    | Dec at the pointing center                                 | deg  | \*           |         |
+| ppc_pa           | float    | Position angle of the pointing center                      | deg  | \*           |         |
+| ppc_resolution   | str      | Resolution mode of the pointing (`L` or `M`)               |      | \*           |         |
+| ppc_priority     | float    | Sum of the priority weights of the pointing                |      | \*           |         |
+| input_catalog_id | int      | `input_catalog_id` in the `input_catalog` table.           |      |              |         |
+| created_at       | datetime | The date and time in UTC when the record was created.      |      |              |         |
+| updated_at       | datetime | The date and time in UTC when the record was last updated. |      |              |         |
+
+[^1]: Required when inserted by using the [CLI tool](../reference/cli.md) or equivalent functions.
+
+## Unique constraint
+
+- `user_pointing_id` is the primary key of the table and set as an auto-increment column.
+
+## Foreign key constraints
+
+- `input_catalog_id` is a foreign key constraint that references the `input_catalog_id` in the `input_catalog` table.

docs/tbls/README.md

@@ -7,8 +7,9 @@
 | [public.filter_name](public.filter_name.md) | 4 |  | BASE TABLE |
 | [public.pfs_arm](public.pfs_arm.md) | 4 |  | BASE TABLE |
 | [public.proposal_category](public.proposal_category.md) | 5 |  | BASE TABLE |
-| [public.input_catalog](public.input_catalog.md) | 8 |  | BASE TABLE |
+| [public.input_catalog](public.input_catalog.md) | 9 |  | BASE TABLE |
 | [public.target_type](public.target_type.md) | 5 |  | BASE TABLE |
+| [public.user_pointing](public.user_pointing.md) | 10 |  | BASE TABLE |
 | [public.proposal](public.proposal.md) | 14 |  | BASE TABLE |
 | [public.sky](public.sky.md) | 14 |  | BASE TABLE |
 | [public.fluxstd](public.fluxstd.md) | 61 |  | BASE TABLE |
@@ -54,11 +55,18 @@
 | public.q3c_poly_query | bool | double precision, double precision, polygon | FUNCTION |
 | public.q3c_poly_query | bool | real, real, polygon | FUNCTION |
 
+## Enums
+
+| Name | Values |
+| ---- | ------- |
+| public.resolutionmode | L, M |
+
 ## Relations
 
 ```mermaid
 erDiagram
 
+"public.user_pointing" }o--|| "public.input_catalog" : "FOREIGN KEY (input_catalog_id) REFERENCES input_catalog(input_catalog_id)"
 "public.proposal" }o--o| "public.proposal_category" : "FOREIGN KEY (proposal_category_id) REFERENCES proposal_category(proposal_category_id)"
 "public.sky" }o--|| "public.input_catalog" : "FOREIGN KEY (input_catalog_id) REFERENCES input_catalog(input_catalog_id)"
 "public.sky" }o--o| "public.target_type" : "FOREIGN KEY (target_type_id) REFERENCES target_type(target_type_id)"
@@ -109,6 +117,7 @@ erDiagram
   varchar_16_ upload_id
   boolean active
   boolean is_classical
+  boolean is_user_pointing
   timestamp_without_time_zone created_at
   timestamp_without_time_zone updated_at
 }
@@ -119,6 +128,18 @@ erDiagram
   timestamp_without_time_zone created_at
   timestamp_without_time_zone updated_at
 }
+"public.user_pointing" {
+  bigint user_pointing_id
+  varchar ppc_code
+  double_precision ppc_ra
+  double_precision ppc_dec
+  double_precision ppc_pa
+  resolutionmode ppc_resolution
+  double_precision ppc_priority
+  integer input_catalog_id FK
+  timestamp_without_time_zone created_at
+  timestamp_without_time_zone updated_at
+}
 "public.proposal" {
   varchar proposal_id
   varchar group_id

docs/tbls/public.cluster.md

@@ -121,6 +121,7 @@ erDiagram
   varchar_16_ upload_id
   boolean active
   boolean is_classical
+  boolean is_user_pointing
   timestamp_without_time_zone created_at
   timestamp_without_time_zone updated_at
 }

docs/tbls/public.fluxstd.md

@@ -183,6 +183,7 @@ erDiagram
   varchar_16_ upload_id
   boolean active
   boolean is_classical
+  boolean is_user_pointing
   timestamp_without_time_zone created_at
   timestamp_without_time_zone updated_at
 }

docs/tbls/public.input_catalog.md

@@ -6,12 +6,13 @@
 
 | Name | Type | Default | Nullable | Children | Parents | Comment |
 | ---- | ---- | ------- | -------- | -------- | ------- | ------- |
-| input_catalog_id | integer |  | false | [public.sky](public.sky.md) [public.fluxstd](public.fluxstd.md) [public.target](public.target.md) [public.cluster](public.cluster.md) |  | Unique identifier for input catalogs |
+| input_catalog_id | integer |  | false | [public.user_pointing](public.user_pointing.md) [public.sky](public.sky.md) [public.fluxstd](public.fluxstd.md) [public.target](public.target.md) [public.cluster](public.cluster.md) |  | Unique identifier for input catalogs |
 | input_catalog_name | varchar |  | false |  |  | Name of the input catalog (e.g., Gaia DR2, HSC-SSP PDR3, etc.) |
 | input_catalog_description | varchar |  | true |  |  | Description of the input catalog |
 | upload_id | varchar(16) |  | true |  |  | A 8-bit hex string (16 characters) assigned at the submission of the target list (default: empty string) |
 | active | boolean |  | true |  |  | Flag to indicate if the input catalog is active (default: True) |
 | is_classical | boolean |  | true |  |  | True if the classical mode is requested |
+| is_user_pointing | boolean |  | true |  |  | True if user-defined pointings are provided |
 | created_at | timestamp without time zone | timezone('utc'::text, CURRENT_TIMESTAMP) | true |  |  | The date and time in UTC when the record was created |
 | updated_at | timestamp without time zone |  | true |  |  | The date and time in UTC when the record was last updated |
 
@@ -32,6 +33,7 @@
 ```mermaid
 erDiagram
 
+"public.user_pointing" }o--|| "public.input_catalog" : "FOREIGN KEY (input_catalog_id) REFERENCES input_catalog(input_catalog_id)"
 "public.sky" }o--|| "public.input_catalog" : "FOREIGN KEY (input_catalog_id) REFERENCES input_catalog(input_catalog_id)"
 "public.fluxstd" }o--|| "public.input_catalog" : "FOREIGN KEY (input_catalog_id) REFERENCES input_catalog(input_catalog_id)"
 "public.target" }o--|| "public.input_catalog" : "FOREIGN KEY (input_catalog_id) REFERENCES input_catalog(input_catalog_id)"
@@ -44,6 +46,19 @@ erDiagram
   varchar_16_ upload_id
   boolean active
   boolean is_classical
+  boolean is_user_pointing
+  timestamp_without_time_zone created_at
+  timestamp_without_time_zone updated_at
+}
+"public.user_pointing" {
+  bigint user_pointing_id
+  varchar ppc_code
+  double_precision ppc_ra
+  double_precision ppc_dec
+  double_precision ppc_pa
+  resolutionmode ppc_resolution
+  double_precision ppc_priority
+  integer input_catalog_id FK
   timestamp_without_time_zone created_at
   timestamp_without_time_zone updated_at
 }

docs/tbls/public.sky.md

@@ -36,9 +36,9 @@
 | ---- | ---------- |
 | sky_pkey | CREATE UNIQUE INDEX sky_pkey ON public.sky USING btree (sky_id) |
 | sky_obj_id_input_catalog_id_version_key | CREATE UNIQUE INDEX sky_obj_id_input_catalog_id_version_key ON public.sky USING btree (obj_id, input_catalog_id, version) |
+| ix_sky_version | CREATE INDEX ix_sky_version ON public.sky USING btree (version) |
 | sky_q3c_ang2ipix_idx | CREATE INDEX sky_q3c_ang2ipix_idx ON public.sky USING btree (q3c_ang2ipix(ra, "dec")) |
 | ix_sky_input_catalog_id | CREATE INDEX ix_sky_input_catalog_id ON public.sky USING btree (input_catalog_id) |
-| ix_sky_version | CREATE INDEX ix_sky_version ON public.sky USING btree (version) |
 
 ## Relations
 
@@ -78,6 +78,7 @@ erDiagram
   varchar_16_ upload_id
   boolean active
   boolean is_classical
+  boolean is_user_pointing
   timestamp_without_time_zone created_at
   timestamp_without_time_zone updated_at
 }

docs/tbls/public.target.md

@@ -217,6 +217,7 @@ erDiagram
   varchar_16_ upload_id
   boolean active
   boolean is_classical
+  boolean is_user_pointing
   timestamp_without_time_zone created_at
   timestamp_without_time_zone updated_at
 }