feat: Handle balances when using Bank Sync (#108)

Allows the accounts to be reconciled when doing the first bank sync. This happens when the expected balance of the account does not match the final balance of the imported transactions, because old transactions are not imported.

This should only affect the first import using the bank sync, or if all transactions are deleted and a new bank sync is done.

Either way, it manages to reproduce 1:1 what we find on the simpleFin demo dataset, when importing from Actual or actualpy.

Closes #29

actual/__init__.py

@@ -37,9 +37,11 @@
 from actual.migrations import js_migration_statements
 from actual.protobuf_models import HULC_Client, Message, SyncRequest
 from actual.queries import (
+    create_transaction,
     get_account,
     get_accounts,
     get_or_create_clock,
+    get_or_create_payee,
     get_ruleset,
     get_transactions,
     reconcile_transaction,
@@ -469,7 +471,9 @@ def run_rules(self):
         transactions = get_transactions(self.session, is_parent=True)
         ruleset.run(transactions)
 
-    def _run_bank_sync_account(self, acct: Accounts, start_date: datetime.date) -> List[Transactions]:
+    def _run_bank_sync_account(
+        self, acct: Accounts, start_date: datetime.date, is_first_sync: bool
+    ) -> List[Transactions]:
         sync_method = acct.account_sync_source
         account_id = acct.account_id
         requisition_id = acct.bank.bank_id if sync_method == "goCardless" else None
@@ -500,6 +504,20 @@ def _run_bank_sync_account(self, acct: Accounts, start_date: datetime.date) -> L
             )
             if reconciled.changed():
                 imported_transactions.append(reconciled)
+        if is_first_sync:
+            current_balance = acct.balance
+            # actual uses 'startingBalance', that already comes in cents and should be enough for our purposes
+            # https://github.com/actualbudget/actual/blob/f09f4af667ddd57e031dcdb0d428ae935aa2afad/packages/loot-core/src/server/accounts/sync.ts#L740-L752
+            expected_balance = new_transactions_data.data.balance
+            balance_to_use = expected_balance - current_balance
+            if balance_to_use:
+                payee = None if acct.offbudget else get_or_create_payee(self.session, "Starting Balance")
+                # get date from the oldest transaction. There seems to be a bug here, and it gets the youngest transaction.
+                oldest_date = new_transactions[-1].date if new_transactions else datetime.date.today()
+                reconciled_transaction = create_transaction(
+                    self.session, oldest_date, acct, payee, amount=balance_to_use
+                )
+                imported_transactions.insert(0, reconciled_transaction)
         return imported_transactions
 
     def run_bank_sync(
@@ -509,6 +527,10 @@ def run_bank_sync(
         Runs the bank synchronization for the selected account. If missing, all accounts are synchronized. If a
         start_date is provided, is used as a reference, otherwise, the last timestamp of each account will be used. If
         the account does not have any transaction, the last 90 days are considered instead.
+
+        If the `start_date` is not provided and the account does not have any transaction, a reconcile transaction will
+        be generated to match the expected balance of the account. This would correct the account balance with the
+        remote one.
         """
         # if no account is provided, sync all of them, otherwise just the account provided
         if account is None:
@@ -518,7 +540,8 @@ def run_bank_sync(
             accounts = [account]
         imported_transactions = []
 
-        default_start_date = start_date
+        default_start_date: datetime.date = start_date
+        is_first_sync: bool = False
         for acct in accounts:
             sync_method = acct.account_sync_source
             account_id = acct.account_id
@@ -532,7 +555,8 @@ def run_bank_sync(
                 if all_transactions:
                     default_start_date = all_transactions[0].get_date()
                 else:
+                    is_first_sync = True
                     default_start_date = datetime.date.today() - datetime.timedelta(days=90)
-            transactions = self._run_bank_sync_account(acct, default_start_date)
+            transactions = self._run_bank_sync_account(acct, default_start_date, is_first_sync)
             imported_transactions.extend(transactions)
         return imported_transactions

actual/api/bank_sync.py

@@ -65,16 +65,16 @@ class Balance(BaseModel):
 
 class TransactionItem(BaseModel):
     transaction_id: str = Field(..., alias="transactionId")
-    booking_date: str = Field(..., alias="bookingDate")
     booked: bool = True
-    value_date: str = Field(..., alias="valueDate")
     transaction_amount: BankSyncAmount = Field(..., alias="transactionAmount")
     # this field will come as either debtorName or creditorName, depending on if it's a debt or credit
     payee: str = Field(None, validation_alias=AliasChoices("debtorName", "creditorName"))
     payee_account: Optional[DebtorAccount] = Field(
         None, validation_alias=AliasChoices("debtorAccount", "creditorAccount")
     )
-    date: datetime.date
+    # 'bookingDate' and 'valueDate' can be null or not existing in some goCardless data. Actual does not use them.
+    # Therefore, we just use them as fallbacks for the date, that should theoretically always exist.
+    date: datetime.date = Field(..., validation_alias=AliasChoices("date", "bookingDate", "valueDate"))
     remittance_information_unstructured: str = Field(None, alias="remittanceInformationUnstructured")
     remittance_information_unstructured_array: List[str] = Field(
         default_factory=list, alias="remittanceInformationUnstructuredArray"
@@ -97,7 +97,7 @@ def notes(self):
         notes = self.remittance_information_unstructured or ", ".join(
             self.remittance_information_unstructured_array or []
         )
-        return notes.strip()
+        return notes.strip().replace("#", "##")
 
 
 class Transactions(BaseModel):
@@ -118,6 +118,10 @@ class BankSyncTransactionData(BaseModel):
     iban: Optional[str] = None
     institution_id: Optional[str] = Field(None, alias="institutionId")
 
+    @property
+    def balance(self) -> decimal.Decimal:
+        return decimal.Decimal(self.starting_balance) / 100
+
 
 class BankSyncErrorData(BaseModel):
     error_type: str

actual/queries.py

@@ -605,7 +605,8 @@ def get_budgets(
     :param month: month to get budgets for, as a date for that month. Use `datetime.date.today()` if you want the budget
                   for current month
     :param category: category to filter for the budget. By default, the query looks for all budgets.
-    :return: list of budgets
+    :return: list of budgets. It's important to note that budgets will only exist if they are actively set beforehand.
+             When the frontend shows a budget as 0.00, it might not be returned by this method.
     """
     table = _get_budget_table(s)
     query = select(table).options(joinedload(table.category))
@@ -630,7 +631,8 @@ def get_budget(
     :param month: month to get budgets for, as a date for that month. Use `datetime.date.today()` if you want the budget
                   for current month.
     :param category: category to filter for the budget.
-    :return: return budget matching the month and category. If not found, returns `None`.
+    :return: returns the budget matching the month and category. If not found, returns `None`. If the budget is not
+             set via frontend, it will show as 0.00, but this function will still return `None`.
     """
     budgets = get_budgets(s, month, category)
     return budgets[0] if budgets else None

tests/test_bank_sync.py

@@ -17,11 +17,11 @@
             "balanceType": "expected",
             "lastChangeDateTime": "2024-06-13T14:56:06.092039915Z",
             "referenceDate": "2024-06-13",
-            "balanceAmount": {"amount": "0.00", "currency": "EUR"},
+            "balanceAmount": {"amount": "1.49", "currency": "EUR"},
         }
     ],
     "institutionId": "My Bank",
-    "startingBalance": 0,
+    "startingBalance": 149,
     "transactions": {
         "all": [
             {
@@ -72,23 +72,35 @@ def create_accounts(session, protocol: str):
     return bank
 
 
-@pytest.fixture
-def set_mocks(mocker):
-    # call for validate
+def generate_bank_sync_data(mocker, starting_balance: int = None):
+    response_full = copy.deepcopy(response)
+    if starting_balance:
+        response_full["startingBalance"] = starting_balance
     response_empty = copy.deepcopy(response)
     response_empty["transactions"]["all"] = []
     mocker.patch.object(Session, "get").return_value = RequestsMock({"status": "ok", "data": {"validated": True}})
     main_mock = mocker.patch.object(Session, "post")
     main_mock.side_effect = [
         RequestsMock({"status": "ok", "data": {"configured": True}}),
-        RequestsMock({"status": "ok", "data": response}),
+        RequestsMock({"status": "ok", "data": response_full}),
         RequestsMock({"status": "ok", "data": {"configured": True}}),  # in case it gets called again
         RequestsMock({"status": "ok", "data": response_empty}),
     ]
     return main_mock
 
 
-def test_full_bank_sync_go_cardless(session, set_mocks):
+@pytest.fixture
+def bank_sync_data_match(mocker):
+    # call for validate
+    return generate_bank_sync_data(mocker)
+
+
+@pytest.fixture
+def bank_sync_data_no_match(mocker):
+    return generate_bank_sync_data(mocker, 2500)
+
+
+def test_full_bank_sync_go_cardless(session, bank_sync_data_match):
     with Actual(token="foo") as actual:
         actual._session = session
         create_accounts(session, "goCardless")
@@ -114,10 +126,10 @@ def test_full_bank_sync_go_cardless(session, set_mocks):
         new_imported_transactions = actual.run_bank_sync()
         assert new_imported_transactions == []
         # assert that the call date is correctly set
-        assert set_mocks.call_args_list[3][1]["json"]["startDate"] == "2024-06-13"
+        assert bank_sync_data_match.call_args_list[3][1]["json"]["startDate"] == "2024-06-13"
 
 
-def test_full_bank_sync_go_simplefin(session, set_mocks):
+def test_full_bank_sync_go_simplefin(session, bank_sync_data_match):
     with Actual(token="foo") as actual:
         actual._session = session
         create_accounts(session, "simplefin")
@@ -138,6 +150,19 @@ def test_full_bank_sync_go_simplefin(session, set_mocks):
         assert imported_transactions[1].notes == "Payment"
 
 
+def test_bank_sync_with_starting_balance(session, bank_sync_data_no_match):
+    with Actual(token="foo") as actual:
+        actual._session = session
+        create_accounts(session, "simplefin")
+        # now try to run the bank sync
+        imported_transactions = actual.run_bank_sync("Bank")
+        assert len(imported_transactions) == 3
+        # first transaction should be the amount
+        assert imported_transactions[0].get_date() == datetime.date(2024, 6, 13)
+        # final amount is 2500 - (926 - 777) = 2351
+        assert imported_transactions[0].get_amount() == decimal.Decimal("23.51")
+
+
 def test_bank_sync_unconfigured(mocker, session):
     mocker.patch.object(Session, "get").return_value = RequestsMock({"status": "ok", "data": {"validated": True}})
     main_mock = mocker.patch.object(Session, "post")