Compare realtime schedule

.github/workflows/cta_data_downloads.yml

@@ -1,6 +1,12 @@
 name: Automate CTA schedule and realtime downloads
 
 on:
+  workflow_dispatch:
+  push:
+    branches:
+    - 'automate-schedule-downloads'
+    - 'date-range-downloads'
+    - 'compare-realtime-schedule'
 
   schedule:
     # Run every day at 12:30pm CST which is 5:30pm UTC
@@ -11,9 +17,11 @@ env:
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
+# Changing ubuntu to macos might resolve some timeout issues
+# See https://github.com/actions/runner-images/issues/6680
 jobs:
   download-cta-schedule-data:
-    runs-on: ubuntu-latest
+    runs-on: macos-latest
 
     steps:
       - uses: actions/checkout@v3
@@ -29,40 +37,60 @@ jobs:
           python -c 'from scrape_data.cta_data_downloads import save_cta_zip; \
            save_cta_zip()' \
            $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY
-
 
-  # save-schedule-daily-summary:
-  #   runs-on: ubuntu-latest
+  save-schedule-daily-summary:
+    runs-on: macos-latest
 
-  #   steps:
-  #     - uses: actions/checkout@v3
+    steps:
+      - uses: actions/checkout@v3
 
-  #     - uses: actions/setup-python@v4
-  #       with:
-  #         python-version: ${{ env.PYTHON_VERSION }}
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
 
-  #     - name: 'Save schedule summaries'
-  #       run: |
-  #         pip install -r requirements.txt
-  #         python -c 'from scrape_data.cta_data_downloads import save_sched_daily_summary; \
-  #          save_sched_daily_summary()' $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY
+      - name: 'Save schedule summaries'
+      # Test with no date and with date range
+        run: |
+          pip install -r requirements.txt
+          python -c 'from scrape_data.cta_data_downloads import save_sched_daily_summary; \
+           save_sched_daily_summary()' $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY        
 
+  save-realtime-daily-summary:
+    runs-on: macos-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
 
-  # save-realtime-daily-summary:
-  #   runs-on: ubuntu-latest
+      - name: 'Save realtime summaries'
 
-  #   steps:
-  #     - uses: actions/checkout@v3
+        run: |
+          pip install -r requirements.txt
+
+          python -c 'from scrape_data.cta_data_downloads import save_realtime_daily_summary; \
+           save_realtime_daily_summary()' $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY
+
+
+  save-frontend-map-json:
+    runs-on: macos-latest
+    needs: [save-realtime-daily-summary, save-schedule-daily-summary]
+    strategy:
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v3
 
-  #     - uses: actions/setup-python@v4
-  #       with:
-  #         python-version: ${{ env.PYTHON_VERSION }}
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
 
-  #     - name: 'Save realtime summaries'
+      - name: 'Save data.json for frontend'
 
-  #       run: |
-  #         pip install -r requirements.txt
+        run: |
+          pip install -r requirements.txt
 
-  #         python -c 'from scrape_data.cta_data_downloads import save_realtime_daily_summary; \
-  #          save_realtime_daily_summary()' $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY
-
+          python -c 'from scrape_data.cta_data_downloads import compare_realtime_sched; \
+           compare_realtime_sched()' $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY
+      

.github/workflows/transitfeeds-backfill-s3.yml

@@ -0,0 +1,38 @@
+name: Download transitfeeds.com zipfiles and save to s3
+
+on: 
+    workflow_dispatch:
+        inputs:
+            start_date:
+                description: 'Start date in YYYY-MM-DD format'
+                required: false
+                type: string
+            end_date:
+                description: 'End date in YYYY-MM-DD format e.g. 2023-05-20'
+                required: false
+                type: string
+
+env: 
+  PYTHON_VERSION: 3.10.6
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+jobs:
+  save-transitfeeds-data:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Download and save transitfeeds.com schedule data
+
+        run: |
+          pip install -r requirements.txt
+          python -c 'from scrape_data.cta_data_downloads import save_transitfeeds_zip; \
+          save_transitfeeds_zip(start_date=${{ inputs.start_date }}, end_date=${{ inputs.end_date}})' \
+          $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY
+

data_analysis/compare_scheduled_and_rt.py

@@ -314,24 +314,33 @@ def build_summary(
         )
     return summary
 
-
-def main(freq: str = 'D') -> Tuple[List[dict],pd.DataFrame, pd.DataFrame]:
-    """Calculate the summary by route and day across multiple schedule versions
+    
+def create_GTFS_data_list(schedule_feeds: List[dict]) -> dict:
+    """ Create list of GTFS data for each schedule version
 
     Args:
-        freq (str): Frequency of aggregation. Defaults to Daily.
+        schedule_feeds (List[dict]): List of dictionaries with the keys
+            'schedule_version', 'feed_start_date', and 'feed_end_date'.
+
     Returns:
-        pd.DataFrame: A DataFrame of every day in the specified data with
-        scheduled and observed count of trips.
-        pd.DataFrame: A DataFrame summary across
-            versioned schedule comparisons.
+        dict: A dictionary with keys 'GTFS_data_list' and 'schedule_data_list'.
+            'GTFS_data_list' is a list of dictionaries with the keys 'schedule_version'
+            and 'data', which is the extracted data from the GTFS zip file.
+            'schedule_data_list' is a list of dictionaries with the same keys as 'GTFS_data_list', 
+            except that 'data' here is the route_daily_summary.
     """
-    schedule_feeds = create_schedule_list(month=5, year=2022)
-
+
+    GTFS_data_list = []
     schedule_data_list = []
     pbar = tqdm(schedule_feeds)
     for feed in pbar:
         schedule_version = feed["schedule_version"]
+        # Files with .zip suffix come from the CTA directly. 
+        # Otherwise, they come from transitfeeds.com 
+        if schedule_version.endswith('.zip'):
+            cta_download = True
+        else:
+            cta_download = False
         pbar.set_description(
             f"Generating daily schedule data for "
             f"schedule version {schedule_version}"
@@ -340,15 +349,18 @@ def main(freq: str = 'D') -> Tuple[List[dict],pd.DataFrame, pd.DataFrame]:
             f"\nDownloading zip file for schedule version "
             f"{schedule_version}"
         )
-        CTA_GTFS = static_gtfs_analysis.download_zip(schedule_version)
+        CTA_GTFS, _ = static_gtfs_analysis.download_zip(schedule_version)
         logger.info("\nExtracting data")
         data = static_gtfs_analysis.GTFSFeed.extract_data(
             CTA_GTFS,
-            version_id=schedule_version
+            version_id=schedule_version,
+            cta_download=cta_download
         )
         data = static_gtfs_analysis.format_dates_hours(data)
+        GTFS_data_list.append({'fname': schedule_version, 'data': data})
 
         logger.info("\nSummarizing trip data")
+
         trip_summary = static_gtfs_analysis.make_trip_summary(data, 
             pendulum.from_format(feed['feed_start_date'], 'YYYY-MM-DD'), 
             pendulum.from_format(feed['feed_end_date'], 'YYYY-MM-DD'))
@@ -360,8 +372,34 @@ def main(freq: str = 'D') -> Tuple[List[dict],pd.DataFrame, pd.DataFrame]:
 
         schedule_data_list.append(
             {"schedule_version": schedule_version,
-             "data": route_daily_summary}
+                "data": route_daily_summary}
         )
+
+
+    return {
+        'GTFS_data_list': GTFS_data_list,
+        'schedule_data_list': schedule_data_list
+    }
+
+def main(freq: str = 'D', schedule_feeds: List[dict] = None
+    ) -> Tuple[List[dict],pd.DataFrame, pd.DataFrame]:
+    """Calculate the summary by route and day across multiple schedule versions
+
+    Args:
+        freq (str): Frequency of aggregation. Defaults to Daily.
+        schedule_feeds (List[dict]): List of dictionaries with the keys
+            'schedule_version', 'feed_start_date', and 'feed_end_date'.
+    Returns:
+        pd.DataFrame: A DataFrame of every day in the specified data with
+            scheduled and observed count of trips.
+        pd.DataFrame: A DataFrame summary across
+            versioned schedule comparisons.
+    """
+    if schedule_feeds is None:
+        schedule_feeds = create_schedule_list(month=5, year=2022)
+
+    schedule_data_list = create_GTFS_data_list(schedule_feeds)['schedule_data_list']
+
     agg_info = AggInfo(freq=freq)
     combined_long, combined_grouped = combine_real_time_rt_comparison(
         schedule_feeds,