Skip to content

Commit

Permalink
Split processes for bulk and updates of sigel #462
Browse files Browse the repository at this point in the history
  • Loading branch information
@TobiasNx
TobiasNx committed Aug 7, 2023
1 parent fc8c5b1 commit 31d892b
Show file tree
Hide file tree
Showing 4 changed files with 31 additions and 12 deletions.
4 changes: 3 additions & 1 deletion app/transformation/TransformAll.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,9 @@ public static void process(String startOfUpdates, int intervalSize,
final String outputPath, String geoServer) throws IOException {
String dbsOutput = outputPath + "-dbs";
String sigelOutput = outputPath + "-sigel";
TransformSigel.process(startOfUpdates, intervalSize, sigelOutput,
TransformSigel.processBulk(startOfUpdates, intervalSize, sigelOutput,
geoServer);
TransformSigel.processUpdates(startOfUpdates, intervalSize, sigelOutput,
geoServer);
TransformDbs.process(dbsOutput, geoServer);
try (FileWriter resultWriter = new FileWriter(outputPath)) {
Expand Down
19 changes: 14 additions & 5 deletions app/transformation/TransformSigel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,22 +49,30 @@ public class TransformSigel {
"/*[local-name() = 'record']/*[local-name() = 'global']/*[local-name() = 'tag'][@id='008H']/*[local-name() = 'subf'][@id='e']";
static final String DUMP_XPATH = "/" + DUMP_TOP_LEVEL_TAG + "/" + XPATH;

static void process(String startOfUpdates, int intervalSize,
static void processBulk(String startOfUpdates, int intervalSize,
final String outputPath, String geoLookupServer) throws IOException {
final FileOpener splitFileOpener = new FileOpener();
final FileOpener dumpOpener = new FileOpener();
final FileOpener dumpOpener = new FileOpener();
PicaDecoder picaDecoder = new PicaDecoder();
picaDecoder.setNormalizeUTF8(true);
JsonEncoder encodeJson = new JsonEncoder();
encodeJson.setPrettyPrinting(true);
dumpOpener//
.setReceiver(new LineReader())//
.setReceiver(new PicaDecoder())//
.setReceiver(picaDecoder)//
.setReceiver(new Metafix("conf/fix-sigel.fix"))//
.setReceiver(TransformAll.fixEnriched(geoLookupServer))//
.setReceiver(encodeJson)//
.setReceiver(TransformAll.esBulk())//
.setReceiver(new ObjectWriter<>(outputPath));
dumpOpener.process(TransformAll.DATA_INPUT_DIR + "sigil.dat");
dumpOpener.closeStream();
}

static void processUpdates(String startOfUpdates, int intervalSize,
final String outputPath, String geoLookupServer) throws IOException {
final FileOpener splitFileOpener = new FileOpener();
JsonEncoder encodeJson = new JsonEncoder();
encodeJson.setPrettyPrinting(true);
ObjectWriter objectWriter = new ObjectWriter<>(outputPath);
objectWriter.setAppendIfFileExists(true);
splitFileOpener//
Expand All @@ -84,7 +92,8 @@ static void process(String startOfUpdates, int intervalSize,
.collect(Collectors.toList()).forEach(path -> {
splitFileOpener.process(path.toString());
});

splitFileOpener.closeStream();


}

Expand Down
10 changes: 7 additions & 3 deletions test/transformation/output/enriched-test.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,12 +65,16 @@
{"index":{"_index":"organisations","_type":"organisation","_id":"http://lobid.org/organisations/DBS-DB675#!"}}
{"rs":"032540021021","type":"Library","classification":{"id":"http://purl.org/lobid/libtype#n81","type":"Concept","label":{"de":"Wissenschaftliche Spezialbibliothek","en":"Academic Special Library"}},"mainEntityOfPage":{"id":"http://lobid.org/organisations/DBS-DB675#!"},"@context":"http://lobid.org/organisations/context.jsonld","url":"http://www.stadtarchiv-hildesheim.de","provides":"https://webopac.stadt-hildesheim.de/libero/WebOpac.cls","name":"Roemer-Museum, Bibliothek","location":[{"address":{"streetAddress":"Am Steine 1/2","addressLocality":"Hildesheim","postalCode":"31134","addressCountry":"DE","type":"PostalAddress"},"openingHoursSpecification":{"description":"Di. Mi.: 09.00 - 16.00 Uhr. Do: 09.00 - 18.00 Uhr","type":"OpeningHoursSpecification"},"type":"Place"}],"id":"http://lobid.org/organisations/DBS-DB675#!","fundertype":{"id":"http://purl.org/lobid/fundertype#n04","type":"Concept","label":{"de":"Gemeinde","en":"Commune"}},"collects":{"type":"Collection","extent":{"id":"http://purl.org/lobid/stocksize#n07","type":"Concept","label":{"de":"30.001 - 100.000","en":"30,001 - 100,000"}}},"dbsID":"DB675"}
{"index":{"_index":"organisations","_type":"organisation","_id":"http://lobid.org/organisations/DE-4209#!"}}
{"rs":"145220180180","note":["Die SAXONIA-FREIBERG-STIFTUNG erforscht, pflegt und fördert das berg- und hüttenmännische Brauchtum und die berg- und hüttenmännische Frömmigkeitstradition in der Region Mittelsachsen. Seit 2020 erfolgt der Aufbau eines zentralen Archivs zur Pflege und Erhaltung des berg- und hüttenmännischen Brauchtums, zunächst gefördert durch das Entwicklungsprogramm für den ländlichen Raum im Freistaat Sachsen. Das Archiv soll zukünftig sowohl die Bestände der Stiftung selbst als auch die interessierter berg- und hüttenmännischer Traditionsvereine aus dem Landkreis Mittelsachsen beherbergen."],"telephone":"00Deutschland 3731 1600","type":"Archive","classification":{"id":"http://purl.org/lobid/libtype#n56","type":"Concept","label":{"de":"Archiv (Parlament, Partei, Stiftung oder Verband)","en":"Archive (Parliament, Party, Foundation or Federation)"}},"mainEntityOfPage":{"id":"http://lobid.org/organisations/DE-4209#!","dateCreated":"11-05-23","dateModified":"11-05-23"},"@context":"http://lobid.org/organisations/context.jsonld","url":"https://saxonia-freiberg-stiftung.de/","picaPPN":"1289093989","name":"Archiv SAXONIA-FREIBERG-STIFTUNG","containedIn":"http://sws.geonames.org/6550757/","location":[{"address":{"streetAddress":"Chemnitzer Straße 8","addressLocality":"Freiberg","postalCode":"09599","addressCountry":"DE","type":"PostalAddress"},"openingHoursSpecification":{"description":"Mo-Do 8.00-16.00 und Fr 8.00-14.00","comment":"Bislang ist das Archiv nicht öffentlich zugänglich, da es sich im Aufbau befindet. Die Stiftung kann aber zu deren Geschäftszeiten besucht werden.","type":"OpeningHoursSpecification"},"geo":{"lat":"50.91358","lon":"13.33452"},"type":"Place"}],"id":"http://lobid.org/organisations/DE-4209#!","isil":"DE-4209","fundertype":{"id":"http://purl.org/lobid/fundertype#n08","type":"Concept","label":{"de":"Körperschaft oder Stiftung des privaten Rechts","en":"Corporate Body or Foundation under Private Law"}},"collects":{"subject":["Geschichte des VEB Bergbau- und Hüttenkombinats „Albert Funk“ Freiberg 1961 – 1990","Geschichte der Saxonia-Freiberg-Stiftung 1993 – Gegenwart","Berg- und hüttenmännische Traditionsvereine in der Region Mittelsachsen"]},"email":"mailto:[email protected]","sameAs":["https://ld.zdb-services.de/resource/organisations/DE-4209"]}
{"rs":"145220180180","note":["Die SAXONIA-FREIBERG-STIFTUNG erforscht, pflegt und fördert das berg- und hüttenmännische Brauchtum und die berg- und hüttenmännische Frömmigkeitstradition in der Region Mittelsachsen. Seit 2020 erfolgt der Aufbau eines zentralen Archivs zur Pflege und Erhaltung des berg- und hüttenmännischen Brauchtums, zunächst gefördert durch das Entwicklungsprogramm für den ländlichen Raum im Freistaat Sachsen. Das Archiv soll zukünftig sowohl die Bestände der Stiftung selbst als auch die interessierter berg- und hüttenmännischer Traditionsvereine aus dem Landkreis Mittelsachsen beherbergen."],"telephone":"00Deutschland 3731 1600","type":"Archive","classification":{"id":"http://purl.org/lobid/libtype#n56","type":"Concept","label":{"de":"Archiv (Parlament, Partei, Stiftung oder Verband)","en":"Archive (Parliament, Party, Foundation or Federation)"}},"mainEntityOfPage":{"id":"http://lobid.org/organisations/DE-4209#!","dateCreated":"11-05-23","dateModified":"11-05-23"},"@context":"http://lobid.org/organisations/context.jsonld","url":"https://saxonia-freiberg-stiftung.de/","picaPPN":"1289093989","name":"Archiv SAXONIA-FREIBERG-STIFTUNG","containedIn":"http://sws.geonames.org/6550757/","location":[{"address":{"streetAddress":"Chemnitzer Straße 8","addressLocality":"Freiberg","postalCode":"09599","addressCountry":"DE","type":"PostalAddress"},"openingHoursSpecification":{"description":"Mo-Do 8.00-16.00 und Fr 8.00-14.00","comment":"Bislang ist das Archiv nicht öffentlich zugänglich, da es sich im Aufbau befindet. Die Stiftung kann aber zu deren Geschäftszeiten besucht werden.","type":"OpeningHoursSpecification"},"geo":{"lat":"50.91358","lon":"13.33452"},"type":"Place"}],"id":"http://lobid.org/organisations/DE-4209#!","isil":"DE-4209","fundertype":{"id":"http://purl.org/lobid/fundertype#n08","type":"Concept","label":{"de":"Körperschaft oder Stiftung des privaten Rechts","en":"Corporate Body or Foundation under Private Law"}},"collects":{"subject":["Geschichte des VEB Bergbau- und Hüttenkombinats „Albert Funk“ Freiberg 1961 – 1990","Geschichte der Saxonia-Freiberg-Stiftung 1993 – Gegenwart","Berg- und hüttenmännische Traditionsvereine in der Region Mittelsachsen"]},"email":"mailto:[email protected]","sameAs":["https://ld.zdb-services.de/resource/organisations/DE-4209"]}
{"index":{"_index":"organisations","_type":"organisation","_id":"http://lobid.org/organisations/ZDB-296-ACSJ#!"}}
{"rs":"032410001001","note":["Online-Ressource / Datenbank"],"alternateName":["TIB AV-Portal"],"type":"Collection","classification":{"id":"http://purl.org/lobid/libtype#n89","type":"Concept","label":{"de":"Paket elektronischer Ressourcen","en":"Collection of Electronic Resources"}},"mainEntityOfPage":{"id":"http://lobid.org/organisations/ZDB-296-ACSJ#!","dateCreated":"12-05-23","dateModified":"24-05-23"},"@context":"http://lobid.org/organisations/context.jsonld","url":"https://pubs.acs.org/","picaPPN":"1289289220","name":"ACS Journals Read & Publish [Niedersachsen-Konsortium], American Chemical Society","location":[{"address":{"streetAddress":"Technische Informationsbibliothek (TIB), Niedersachsen-Konsortium","addressLocality":"Hannover","postalCode":"30167","addressCountry":"DE","type":"PostalAddress"},"geo":{"lat":"52.38163","lon":"9.72001"},"type":"Place"}],"id":"http://lobid.org/organisations/ZDB-296-ACSJ#!","isil":"ZDB-296-ACSJ","sigel":"ZDB-296-ACSJ","collects":{"subject":["E-Journals"]},"email":"mailto:[email protected]","sameAs":["https://ld.zdb-services.de/resource/organisations/ZDB-296-ACSJ"]}
{"index":{"_index":"organisations","_type":"organisation","_id":"http://lobid.org/organisations/DE-4210#!"}}
{"rs":"150020000000","note":["Allgemeine Erläuterungen: Unsere Bibliothek ist ist aus dem Nachlass von Dr. Walther Thiede (1931-2011) entstanden. Seine Bibliothek umfasste mehr als 44.000 Bücher/Zeitschriftenbände. Schwerpunkt der Bibliothek ist die Ornithologie i.w.S. Aufgabe unserer Stiftung ist es (neben der Herausgabe einer ornithologischen Fachzeitschrift) die Bestände öffentlich zugänglich zu machen. Seit 2018 katalogisieren wir die Bestände und seitdem ist die Bibliothek auch (nach Anmeldung) für Besucher geöffnet."],"address":{"streetAddress":"Buchenweg 14","addressLocality":"Halle (Saale)","postalCode":"06132","addressCountry":"DE","type":"PostalAddress"},"type":"Library","classification":{"id":"http://purl.org/lobid/libtype#n81","type":"Concept","label":{"de":"Wissenschaftliche Spezialbibliothek","en":"Academic Special Library"}},"mainEntityOfPage":{"id":"http://lobid.org/organisations/DE-4210#!","dateCreated":"16-05-23","dateModified":"23-05-23"},"@context":"http://lobid.org/organisations/context.jsonld","picaPPN":"1289541574","name":"Bibliothek der Dr. Walther-Thiede-Stiftung","containedIn":"http://sws.geonames.org/6551220/","location":[{"address":{"streetAddress":"Alter Markt 14","addressLocality":"Halle (Saale)","postalCode":"06108","addressCountry":"DE","type":"PostalAddress"},"openingHoursSpecification":{"comment":"nach Anmeldung","type":"OpeningHoursSpecification"},"geo":{"lat":"51.48018","lon":"11.96791"},"type":"Place"}],"id":"http://lobid.org/organisations/DE-4210#!","isil":"DE-4210","fundertype":{"id":"http://purl.org/lobid/fundertype#n08","type":"Concept","label":{"de":"Körperschaft oder Stiftung des privaten Rechts","en":"Corporate Body or Foundation under Private Law"}},"sameAs":["https://ld.zdb-services.de/resource/organisations/DE-4210"]}
{"rs":"150020000000","note":["Allgemeine Erläuterungen: Unsere Bibliothek ist ist aus dem Nachlass von Dr. Walther Thiede (1931-2011) entstanden. Seine Bibliothek umfasste mehr als 44.000 Bücher/Zeitschriftenbände. Schwerpunkt der Bibliothek ist die Ornithologie i.w.S. Aufgabe unserer Stiftung ist es (neben der Herausgabe einer ornithologischen Fachzeitschrift) die Bestände öffentlich zugänglich zu machen. Seit 2018 katalogisieren wir die Bestände und seitdem ist die Bibliothek auch (nach Anmeldung) für Besucher geöffnet."],"address":{"streetAddress":"Buchenweg 14","addressLocality":"Halle (Saale)","postalCode":"06132","addressCountry":"DE","type":"PostalAddress"},"type":"Library","classification":{"id":"http://purl.org/lobid/libtype#n81","type":"Concept","label":{"de":"Wissenschaftliche Spezialbibliothek","en":"Academic Special Library"}},"mainEntityOfPage":{"id":"http://lobid.org/organisations/DE-4210#!","dateCreated":"16-05-23","dateModified":"23-05-23"},"@context":"http://lobid.org/organisations/context.jsonld","picaPPN":"1289541574","name":"Bibliothek der Dr. Walther-Thiede-Stiftung","containedIn":"http://sws.geonames.org/6551220/","location":[{"address":{"streetAddress":"Alter Markt 14","addressLocality":"Halle (Saale)","postalCode":"06108","addressCountry":"DE","type":"PostalAddress"},"openingHoursSpecification":{"comment":"nach Anmeldung","type":"OpeningHoursSpecification"},"geo":{"lat":"51.48018","lon":"11.96791"},"type":"Place"}],"id":"http://lobid.org/organisations/DE-4210#!","isil":"DE-4210","fundertype":{"id":"http://purl.org/lobid/fundertype#n08","type":"Concept","label":{"de":"Körperschaft oder Stiftung des privaten Rechts","en":"Corporate Body or Foundation under Private Law"}},"sameAs":["https://ld.zdb-services.de/resource/organisations/DE-4210"]}
{"index":{"_index":"organisations","_type":"organisation","_id":"http://lobid.org/organisations/DE-4211#!"}}
{"rs":"082355007046","telephone":"0049 7452 681 494","type":"Archive","classification":{"id":"http://purl.org/lobid/libtype#n52","type":"Concept","label":{"de":"Archiv (kommunal)","en":"Archive (Municipal)"}},"mainEntityOfPage":{"id":"http://lobid.org/organisations/DE-4211#!","dateCreated":"16-05-23","dateModified":"16-05-23"},"@context":"http://lobid.org/organisations/context.jsonld","url":"https://www.nagold.de/willkommen/Stadtarchiv","picaPPN":"1289547572","name":"Stadtarchiv Nagold","location":[{"address":{"streetAddress":"Grabenstraße 1","addressLocality":"Nagold","postalCode":"72202","addressCountry":"DE","type":"PostalAddress"},"geo":{"lat":"48.55150","lon":"8.72474"},"type":"Place"}],"id":"http://lobid.org/organisations/DE-4211#!","isil":"DE-4211","fundertype":{"id":"http://purl.org/lobid/fundertype#n04","type":"Concept","label":{"de":"Gemeinde","en":"Commune"}},"email":"mailto:[email protected]","sameAs":["https://ld.zdb-services.de/resource/organisations/DE-4211"]}
{"index":{"_index":"organisations","_type":"organisation","_id":"http://lobid.org/organisations/DE-4212#!"}}
{"rs":"059150000000","note":["Eröffnung Lehrmittelverwaltung innerhalb der Schule"],"telephone":"00Deutschland 2381 914960","type":"Archive","classification":{"id":"http://purl.org/lobid/libtype#n59","type":"Concept","label":{"de":"Archiv (Sonstiges)","en":"Archive (Miscellaneous)"}},"mainEntityOfPage":{"id":"http://lobid.org/organisations/DE-4212#!","dateCreated":"22-05-23","dateModified":"22-05-23"},"@context":"http://lobid.org/organisations/context.jsonld","url":"https://das-stein.de","picaPPN":"1290221324","name":"Freiherr-vom-Stein Gymnasium Hamm","location":[{"address":{"streetAddress":"Karl-Koßmann-Str. 2","addressLocality":"Hamm","postalCode":"59071","addressCountry":"DE","type":"PostalAddress"},"openingHoursSpecification":{"description":"Mo-Fr 7.55 - 13.10","type":"OpeningHoursSpecification"},"geo":{"lat":"51.68467","lon":"7.88521"},"type":"Place"}],"id":"http://lobid.org/organisations/DE-4212#!","isil":"DE-4212","fundertype":{"id":"http://purl.org/lobid/fundertype#n02","type":"Concept","label":{"de":"Land","en":"Federal State"}},"collects":{"type":"Collection","extent":{"id":"http://purl.org/lobid/stocksize#n06","type":"Concept","label":{"de":"10.001 - 30.000","en":"10,001 - 30,000"}}},"email":"mailto:[email protected]","sameAs":["https://ld.zdb-servi
{"rs":"059150000000","note":["Eröffnung Lehrmittelverwaltung innerhalb der Schule"],"telephone":"00Deutschland 2381 914960","type":"Archive","classification":{"id":"http://purl.org/lobid/libtype#n59","type":"Concept","label":{"de":"Archiv (Sonstiges)","en":"Archive (Miscellaneous)"}},"mainEntityOfPage":{"id":"http://lobid.org/organisations/DE-4212#!","dateCreated":"22-05-23","dateModified":"22-05-23"},"@context":"http://lobid.org/organisations/context.jsonld","url":"https://das-stein.de","picaPPN":"1290221324","name":"Freiherr-vom-Stein Gymnasium Hamm","location":[{"address":{"streetAddress":"Karl-Koßmann-Str. 2","addressLocality":"Hamm","postalCode":"59071","addressCountry":"DE","type":"PostalAddress"},"openingHoursSpecification":{"description":"Mo-Fr 7.55 - 13.10","type":"OpeningHoursSpecification"},"geo":{"lat":"51.68467","lon":"7.88521"},"type":"Place"}],"id":"http://lobid.org/organisations/DE-4212#!","isil":"DE-4212","fundertype":{"id":"http://purl.org/lobid/fundertype#n02","type":"Concept","label":{"de":"Land","en":"Federal State"}},"collects":{"type":"Collection","extent":{"id":"http://purl.org/lobid/stocksize#n06","type":"Concept","label":{"de":"10.001 - 30.000","en":"10,001 - 30,000"}}},"email":"mailto:[email protected]","sameAs":["https://ld.zdb-services.de/resource/organisations/DE-4212"]}
{"index":{"_index":"organisations","_type":"organisation","_id":"http://lobid.org/organisations/ZDB-18-BLS#!"}}
{"rs":"091620000000","note":["Online-Ressource / Datenbank"],"alternateName":["Beck eLibrary / C.H. Beck Literatur und Sprache (früherer Name)","Beck eLib Literatur&Sprache"],"type":"Collection","classification":{"id":"http://purl.org/lobid/libtype#n89","type":"Concept","label":{"de":"Paket elektronischer Ressourcen","en":"Collection of Electronic Resources"}},"mainEntityOfPage":{"id":"http://lobid.org/organisations/ZDB-18-BLS#!","dateCreated":"22-05-23","dateModified":"22-05-23"},"@context":"http://lobid.org/organisations/context.jsonld","url":"http://elibrary.chbeck.de/index.php","picaPPN":"1290222630","name":"Beck eLibrary / C.H. Beck LSW gesamt","location":[{"address":{"streetAddress":"Verlag C. H. Beck oHG, Wilhelmstr. 9","addressLocality":"München","postalCode":"80801","addressCountry":"DE","type":"PostalAddress"},"geo":{"lat":"48.15921","lon":"11.58276"},"type":"Place"}],"id":"http://lobid.org/organisations/ZDB-18-BLS#!","isil":"ZDB-18-BLS","sigel":"ZDB-18-BLS","collects":{"subject":["E-Books","das Gesamtpaket umfasst die Titel aus ZDB-18-BLA, ZDB-18-BLC, ZDB-18-BLI, ZDB-18-BLF, ZDB-18-BLH, ZDB-18-BLB, ZDB-18-BLE, ZDB-18-BLG"]},"email":"mailto:[email protected]","sameAs":["https://ld.zdb-services.de/resource/organisations/ZDB-18-BLS"]}
{"index":{"_index":"organisations","_type":"organisation","_id":"http://lobid.org/organisations/DE-4213#!"}}
{"type":"Museum","classification":{"id":"http://purl.org/lobid/libtype#n86","type":"Concept","label":{"de":"Museum","en":"http://purl.org/lobid/libtype#n86"}},"mainEntityOfPage":{"id":"http://lobid.org/organisations/DE-4213#!","dateCreated":"25-05-23","dateModified":"25-05-23"},"@context":"http://lobid.org/organisations/context.jsonld","url":"https://www.meiningermuseen.de/pages/die-museen/sammlungen/musikgeschichte.php","picaPPN":"129065686X","name":"Kulturstiftung Meiningen-Eisenach, GB Meininger Museen, Sammlung Musikgeschichte/Max-Reger-Archiv","location":[{"address":{"streetAddress":"Schlossplatz 1","addressLocality":"Meiningen","postalCode":"98617","addressCountry":"DE","type":"PostalAddress"},"geo":{"lat":"50.57046","lon":"10.41258"},"type":"Place"}],"id":"http://lobid.org/organisations/DE-4213#!","isil":"DE-4213","fundertype":{"id":"http://purl.org/lobid/fundertype#n08","type":"Concept","label":{"de":"Körperschaft oder Stiftung des privaten Rechts","en":"Corporate Body or Foundation under Private Law"}},"collects":{"type":"Collection","extent":{"id":"http://purl.org/lobid/stocksize#n04","type":"Concept","label":{"de":"3.001 - 5.000","en":"3,001 - 5,000"}},"subject":["Musikgeschichte"]},"sameAs":["https://ld.zdb-services.de/resource/organisations/DE-4213"]}
Loading

0 comments on commit 31d892b

Please sign in to comment.