Merge pull request #559 from spacious-team/develop

Релиз 2023.2

.gitignore

@@ -24,6 +24,7 @@ investbook/
 !.idea/codeStyles
 !.idea/copyright
 !.idea/inspectionProfiles
+!.idea/encodings.xml
 
 ### NetBeans ###
 /nbproject/private/

README.md

@@ -1,8 +1,8 @@
 [![java-version](https://img.shields.io/badge/java-20-brightgreen?style=flat-square)](https://openjdk.org/)
-[![spring-boot-version](https://img.shields.io/badge/spring--boot-3.0.6-brightgreen?style=flat-square)](https://github.com/spring-projects/spring-boot/releases)
+[![spring-boot-version](https://img.shields.io/badge/spring--boot-3.0.7-brightgreen?style=flat-square)](https://github.com/spring-projects/spring-boot/releases)
 [![hits-of-code](https://img.shields.io/badge/dynamic/json?style=flat-square&color=lightblue&label=hits-of-code&url=https://hitsofcode.com/github/spacious-team/investbook/json?branch=develop&query=$.count)](https://hitsofcode.com/github/spacious-team/investbook/view?branch=develop)
-![github-closed-pull-requests](https://img.shields.io/github/issues-pr-closed/spacious-team/investbook?style=flat-square&color=brightgreen)
-![github-workflow-status](https://img.shields.io/github/actions/workflow/status/spacious-team/investbook/publish-docker.yml?style=flat-square&branch=master)
+[![github-closed-pull-requests](https://img.shields.io/github/issues-pr-closed/spacious-team/investbook?style=flat-square&color=brightgreen)](https://github.com/spacious-team/investbook/pulls?q=is%3Apr+is%3Aclosed)
+[![github-workflow-status](https://img.shields.io/github/actions/workflow/status/spacious-team/investbook/publish-docker.yml?style=flat-square&branch=master)](https://github.com/spacious-team/investbook/actions/workflows/publish-docker.yml)
 [![github-all-releases](https://img.shields.io/github/downloads/spacious-team/investbook/total?style=flat-square&logo=github&color=lightblue)](https://github.com/spacious-team/investbook/releases/latest)
 [![docker-pulls](https://img.shields.io/docker/pulls/spaciousteam/investbook?style=flat-square&logo=docker&color=lightblue&logoColor=white)](https://hub.docker.com/r/spaciousteam/investbook)
 [![telegram-channel](https://img.shields.io/endpoint?style=flat-square&color=2ca5e0&label=news&url=https%3A%2F%2Ftg.sumanjay.workers.dev%2Finvestbook_official)](https://t.me/investbook_official)

docs/dbms-changing.md

@@ -6,6 +6,7 @@
 Если у вас недостаточно опыта, то рекомендуется пропустить этот раздел.
 
 Возможен переход на [MariaDB](https://downloads.mariadb.org/).
+Поддерживаются версии, указанные в документации [Flyway](https://documentation.red-gate.com/fd/mariadb-184127600.html).
 После установки в файле `application-conf.properties` необходимо прописать
 ```
 spring.profiles.active=core,mariadb,conf

pom.xml

@@ -23,12 +23,12 @@
     <parent>
         <groupId>org.springframework.boot</groupId>
         <artifactId>spring-boot-starter-parent</artifactId>
-        <version>3.0.6</version> <!-- Обновлять с учетом https://github.com/spacious-team/investbook/issues/545 -->
+        <version>3.0.7</version> <!-- Обновлять с учетом https://github.com/spacious-team/investbook/issues/545 -->
         <relativePath/> <!-- lookup parent from repository -->
     </parent>
     <groupId>ru.investbook</groupId>
     <artifactId>investbook</artifactId>
-    <version>2023.1</version>
+    <version>2023.2</version>
 
     <name>investbook</name>
     <description>Investor Accounting Book</description>
@@ -62,7 +62,7 @@
 
     <properties>
         <!-- Valid version is (0-255).(0-255).(0-65535) -->
-        <win.msi.version>23.1</win.msi.version>
+        <win.msi.version>23.2</win.msi.version>
         <java.version>20</java.version>
     </properties>
 
@@ -427,7 +427,7 @@
                                     <!-- Defines available bellsoft-liberica java versions from
                                     https://github.com/paketo-buildpacks/java/releases
                                     JRE with ${java.version} will be selected automatically when available -->
-                                    <buildpack>gcr.io/paketo-buildpacks/java:9.8.0</buildpack>
+                                    <buildpack>gcr.io/paketo-buildpacks/java:9.20.0</buildpack>
                                 </buildpacks>
                             </image>
                             <docker>

src/main/java/ru/investbook/parser/tinkoff/SecurityCodeAndIsinTable.java

@@ -20,20 +20,22 @@
 
 import jakarta.validation.constraints.NotNull;
 import lombok.Getter;
+import lombok.RequiredArgsConstructor;
 import org.spacious_team.broker.pojo.SecurityType;
 import org.spacious_team.broker.report_parser.api.AbstractReportTable;
 import org.spacious_team.broker.report_parser.api.BrokerReport;
+import org.spacious_team.table_wrapper.api.OptionalTableColumn;
 import org.spacious_team.table_wrapper.api.PatternTableColumn;
 import org.spacious_team.table_wrapper.api.TableColumn;
 import org.spacious_team.table_wrapper.api.TableHeaderColumn;
 import org.spacious_team.table_wrapper.api.TableRow;
-import org.springframework.util.StringUtils;
 
 import java.math.BigDecimal;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 
+import static org.springframework.util.StringUtils.hasLength;
 import static ru.investbook.parser.tinkoff.SecurityCodeAndIsinTable.SecurityAndCodeTableHeader.*;
 import static ru.investbook.parser.tinkoff.TinkoffBrokerReport.tablesLastRowPattern;
 
@@ -54,8 +56,10 @@ protected SecurityCodeAndIsinTable(BrokerReport report) {
     @Override
     protected Void parseRow(TableRow row) {
         String code = row.getStringCellValueOrDefault(CODE, null);
-        if (StringUtils.hasLength(code) && !code.contains("Код актива")) { // exclude table's empty row
-            codeToIsin.put(code, row.getStringCellValue(ISIN));
+        if (hasLength(code) && !code.contains("Код актива")) { // exclude table's empty row
+            // если колонка ISIN отсутствует, то ISIN используется в отчете вместо кода (SBERP)
+            String isin = row.getStringCellValueOrDefault(ISIN, code);
+            codeToIsin.put(code, isin);
 
             String type = row.getStringCellValueOrDefault(TYPE, "").toLowerCase();
             SecurityType securityType;
@@ -74,7 +78,7 @@ protected Void parseRow(TableRow row) {
             }
 
             String shortName = row.getStringCellValueOrDefault(SHORT_NAME, null);
-            if (StringUtils.hasLength(shortName)) {
+            if (hasLength(shortName)) {
                 shortNameToCode.put(shortName, code);
             }
         }
@@ -105,10 +109,11 @@ public String getCode(String shortName) {
         return Objects.requireNonNull(shortNameToCode.get(shortName), "Не найден код бумаги");
     }
 
+    @RequiredArgsConstructor
     protected enum SecurityAndCodeTableHeader implements TableHeaderColumn {
         SHORT_NAME("Сокращенное", "наименование"),
-        CODE("код", "актива"),
-        ISIN("isin"),
+        CODE("код", "актива"),  // код (SBERP) или ISIN
+        ISIN(optional("isin")), // может отсутствовать, если колонка CODE заполняется ISIN
         TYPE("^Тип$"),
         FACE_VALUE("Номинал");
 
@@ -118,5 +123,9 @@ protected enum SecurityAndCodeTableHeader implements TableHeaderColumn {
         SecurityAndCodeTableHeader(String... words) {
             this.column = PatternTableColumn.of(words);
         }
+
+        static OptionalTableColumn optional(String... words) {
+            return OptionalTableColumn.of(PatternTableColumn.of(words));
+        }
     }
 }

src/main/java/ru/investbook/parser/tinkoff/TinkoffReportTables.java

@@ -40,10 +40,12 @@ public class TinkoffReportTables extends AbstractReportTables<TinkoffBrokerRepor
     private final TinkoffCashTable portfolioCashTable;
     @Getter
     private final TinkoffPortfolioPropertyTable portfolioPropertyTable;
+    private final TinkoffSecurityTransactionTableHelper tinkoffSecurityTransactionTableHelper;
 
     protected TinkoffReportTables(TinkoffBrokerReport report,
                                   TransactionValueAndFeeParser transactionValueAndFeeParser,
-                                  ForeignExchangeRateService foreignExchangeRateService) {
+                                  ForeignExchangeRateService foreignExchangeRateService,
+                                  TinkoffSecurityTransactionTableHelper tinkoffSecurityTransactionTableHelper) {
         super(report);
         this.transactionValueAndFeeParser = transactionValueAndFeeParser;
         this.securityCodeAndIsinTable = new SecurityCodeAndIsinTable(this.report);
@@ -53,6 +55,7 @@ protected TinkoffReportTables(TinkoffBrokerReport report,
         this.securityQuoteTable = WrappingReportTable.of(securityQuoteTables);
         this.portfolioPropertyTable = new TinkoffPortfolioPropertyTable(report, foreignExchangeRateService,
                 portfolioCashTable, securityQuoteTables);
+        this.tinkoffSecurityTransactionTableHelper = tinkoffSecurityTransactionTableHelper;
     }
 
     @Override
@@ -68,7 +71,8 @@ public ReportTable<Security> getSecuritiesTable() {
     @Override
     public ReportTable<AbstractTransaction> getTransactionTable() {
         ReportTable<AbstractTransaction> transactionTable =
-                TinkoffSecurityTransactionTable.of(report, securityCodeAndIsinTable, transactionValueAndFeeParser);
+                TinkoffSecurityTransactionTable.of(report, securityCodeAndIsinTable, transactionValueAndFeeParser,
+                        tinkoffSecurityTransactionTableHelper);
         TinkoffDepositAndWithdrawalTable depositAndWithdrawalTable =
                 new TinkoffDepositAndWithdrawalTable(report, transactionTable, securityCodeAndIsinTable);
         return WrappingReportTable.of(transactionTable, depositAndWithdrawalTable);

src/main/java/ru/investbook/parser/tinkoff/TinkoffReportTablesFactory.java

@@ -25,12 +25,14 @@
 import org.springframework.stereotype.Component;
 import ru.investbook.parser.TransactionValueAndFeeParser;
 import ru.investbook.report.ForeignExchangeRateService;
+import ru.investbook.service.moex.MoexDerivativeCodeService;
 
 @Component
 @RequiredArgsConstructor
 public class TinkoffReportTablesFactory implements ReportTablesFactory {
     private final TransactionValueAndFeeParser transactionValueAndFeeParser;
     private final ForeignExchangeRateService foreignExchangeRateService;
+    private final MoexDerivativeCodeService moexDerivativeCodeService;
 
     @Override
     public boolean canCreate(BrokerReport brokerReport) {
@@ -39,7 +41,9 @@ public boolean canCreate(BrokerReport brokerReport) {
 
     @Override
     public ReportTables create(BrokerReport brokerReport) {
+        TinkoffSecurityTransactionTableHelper tinkoffSecurityTransactionTableHelper =
+                new TinkoffSecurityTransactionTableHelper(moexDerivativeCodeService);
         return new TinkoffReportTables((TinkoffBrokerReport) brokerReport,
-                transactionValueAndFeeParser, foreignExchangeRateService);
+                transactionValueAndFeeParser, foreignExchangeRateService, tinkoffSecurityTransactionTableHelper);
     }
 }

src/main/java/ru/investbook/parser/tinkoff/TinkoffSecurityTransactionTable.java

@@ -21,6 +21,7 @@
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.spacious_team.broker.pojo.SecurityType;
 import org.spacious_team.broker.report_parser.api.AbstractTransaction;
 import org.spacious_team.broker.report_parser.api.DerivativeTransaction;
 import org.spacious_team.broker.report_parser.api.ForeignExchangeTransaction;
@@ -39,37 +40,39 @@
 import java.time.Instant;
 
 import static ru.investbook.parser.tinkoff.TinkoffSecurityTransactionTable.TransactionTableHeader.*;
-import static ru.investbook.parser.tinkoff.TinkoffSecurityTransactionTableHelper.getSecurityId;
-import static ru.investbook.parser.tinkoff.TinkoffSecurityTransactionTableHelper.getSecurityType;
 
 @Slf4j
 public class TinkoffSecurityTransactionTable extends SingleAbstractReportTable<AbstractTransaction> {
 
     private final SecurityCodeAndIsinTable codeAndIsin;
     private final TransactionValueAndFeeParser transactionValueAndFeeParser;
+    private final TinkoffSecurityTransactionTableHelper transactionTableHelper;
 
     public static ReportTable<AbstractTransaction> of(TinkoffBrokerReport report,
                                                       SecurityCodeAndIsinTable codeAndIsin,
-                                                      TransactionValueAndFeeParser transactionValueAndFeeParser) {
+                                                      TransactionValueAndFeeParser transactionValueAndFeeParser,
+                                                      TinkoffSecurityTransactionTableHelper transactionTableHelper) {
         return WrappingReportTable.of(
                 new TinkoffSecurityTransactionTable(
                         "1.1 Информация о совершенных и исполненных сделках",
-                        report, codeAndIsin, transactionValueAndFeeParser),
+                        report, codeAndIsin, transactionValueAndFeeParser, transactionTableHelper),
                 new TinkoffSecurityTransactionTable(
                         "1.2 Информация о неисполненных сделках на конец отчетного периода",
-                        report, codeAndIsin, transactionValueAndFeeParser));
+                        report, codeAndIsin, transactionValueAndFeeParser, transactionTableHelper));
     }
 
     private TinkoffSecurityTransactionTable(String tableNamePrefix,
                                             TinkoffBrokerReport report,
                                             SecurityCodeAndIsinTable codeAndIsin,
-                                            TransactionValueAndFeeParser transactionValueAndFeeParser) {
+                                            TransactionValueAndFeeParser transactionValueAndFeeParser,
+                                            TinkoffSecurityTransactionTableHelper transactionTableHelper) {
         super(report,
                 cell -> cell.startsWith(tableNamePrefix),
                 cell -> TinkoffBrokerReport.tablesLastRowPattern.matcher(cell).lookingAt(),
                 TransactionTableHeader.class);
         this.codeAndIsin = codeAndIsin;
         this.transactionValueAndFeeParser = transactionValueAndFeeParser;
+        this.transactionTableHelper = transactionTableHelper;
     }
 
     @Override
@@ -78,15 +81,16 @@ protected AbstractTransaction parseRow(TableRow row) {
         if (_tradeId == -1) return null;
         String tradeId = String.valueOf(_tradeId);
 
-        int securityId = getSecurityId(row, codeAndIsin, getReport().getSecurityRegistrar());
+        int securityId = transactionTableHelper.getSecurityId(row, codeAndIsin, getReport().getSecurityRegistrar());
         boolean isBuy = row.getStringCellValue(OPERATION).toLowerCase().contains("покупка");
         int count = Math.abs(row.getIntCellValue(COUNT));
         BigDecimal amount = row.getBigDecimalCellValue(AMOUNT).abs();
         amount = isBuy ? amount.negate() : amount;
         count = isBuy ? count : -count;
 
         Instant timestamp = null;
-        AbstractTransaction.AbstractTransactionBuilder<?, ?> builder = switch (getSecurityType(row)) {
+        SecurityType securityType = transactionTableHelper.getSecurityType(row);
+        AbstractTransaction.AbstractTransactionBuilder<?, ?> builder = switch (securityType) {
             case STOCK -> SecurityTransaction.builder()
                     .timestamp(timestamp = getStockAndBondTransactionInstant(row));
             case BOND, STOCK_OR_BOND -> {
@@ -137,7 +141,11 @@ protected AbstractTransaction parseRow(TableRow row) {
     }
 
     private Instant getStockAndBondTransactionInstant(TableRow row) {
-        return getReport().convertToInstant(row.getStringCellValue(SETTLEMENT_DATE));
+        String settlementDate = row.getStringCellValue(SETTLEMENT_DATE);
+        // В старых отчетах одна дата, с января 2023 две: план/факт
+        String[] dates = settlementDate.split("/");
+        String date = (dates.length == 1) ? dates[0] : dates[1];
+        return getReport().convertToInstant(date);
     }
 
     private Instant getDerivativeAndCurrencyPairTransactionInstant(TableRow row) {
@@ -150,7 +158,7 @@ protected enum TransactionTableHeader implements TableHeaderColumn {
         TRADE_ID("номер", "сделки"),
         TRANSACTION_DATE("дата", "заклю", "чения"),
         TRANSACTION_TIME("время"),
-        TYPE("режим", "торгов"),
+        TYPE(optional("режим", "торгов")),
         OPERATION("вид", "сделки"),
         SHORT_NAME("сокращен", "наименова"),
         CODE("код", "актива"),
@@ -177,5 +185,9 @@ protected enum TransactionTableHeader implements TableHeaderColumn {
         TransactionTableHeader(String... words) {
             this.column = PatternTableColumn.of(words);
         }
+
+        static OptionalTableColumn optional(String... words) {
+            return OptionalTableColumn.of(PatternTableColumn.of(words));
+        }
     }
 }

src/main/java/ru/investbook/parser/tinkoff/TinkoffSecurityTransactionTableHelper.java

@@ -18,19 +18,25 @@
 
 package ru.investbook.parser.tinkoff;
 
+import lombok.RequiredArgsConstructor;
 import org.spacious_team.broker.pojo.Security;
 import org.spacious_team.broker.pojo.SecurityType;
 import org.spacious_team.table_wrapper.api.TableRow;
+import org.springframework.stereotype.Component;
 import ru.investbook.parser.SecurityRegistrar;
+import ru.investbook.service.moex.MoexDerivativeCodeService;
 
 import java.math.BigDecimal;
 import java.util.Objects;
 
 import static ru.investbook.parser.tinkoff.TinkoffSecurityTransactionTable.TransactionTableHeader.*;
 
-class TinkoffSecurityTransactionTableHelper {
+@Component
+@RequiredArgsConstructor
+public class TinkoffSecurityTransactionTableHelper {
+    private final MoexDerivativeCodeService moexDerivativeCodeService;
 
-    static int getSecurityId(TableRow row,
+    int getSecurityId(TableRow row,
                              SecurityCodeAndIsinTable codeAndIsin,
                              SecurityRegistrar registrar) {
         String code = row.getStringCellValue(CODE);
@@ -40,7 +46,7 @@ static int getSecurityId(TableRow row,
         return declareSecurity(security, registrar);
     }
 
-    static SecurityType getSecurityType(TableRow row) {
+    SecurityType getSecurityType(TableRow row) {
         BigDecimal accruedInterest = row.getBigDecimalCellValueOrDefault(ACCRUED_INTEREST, BigDecimal.ZERO);
         if (accruedInterest.floatValue() > 1e-3) {
             return SecurityType.BOND;
@@ -56,6 +62,8 @@ static SecurityType getSecurityType(TableRow row) {
         String shortName = row.getStringCellValueOrDefault(SHORT_NAME, "");
         if (shortName != null && shortName.length() == 10 && shortName.charAt(6) == '_') { // USDRUB_TOM
             return SecurityType.CURRENCY_PAIR;
+        } else if (moexDerivativeCodeService.isFuturesCode(shortName) || moexDerivativeCodeService.isOption(shortName)) {
+            return SecurityType.DERIVATIVE;
         }
 
         return SecurityType.STOCK;