BAG2LoaderMain.java

  1. /*
  2.  * Copyright (C) 2021 B3Partners B.V.
  3.  *
  4.  * SPDX-License-Identifier: MIT
  5.  *
  6.  */

  8. package nl.b3p.brmo.bag2.loader.cli;

  10. import static nl.b3p.brmo.bag2.schema.BAG2SchemaMapper.METADATA_TABLE_NAME;
  11. import static nl.b3p.brmo.bag2.schema.BAG2SchemaMapper.Metadata.CURRENT_TECHNISCHE_DATUM;
  12. import static nl.b3p.brmo.bag2.schema.BAG2SchemaMapper.Metadata.FILTER_GEOMETRIE;
  13. import static nl.b3p.brmo.bag2.schema.BAG2SchemaMapper.Metadata.GEMEENTE_CODES;
  14. import static nl.b3p.brmo.bag2.schema.BAG2SchemaMapper.Metadata.STAND_LOAD_TECHNISCHE_DATUM;
  15. import static nl.b3p.brmo.bag2.schema.BAG2SchemaMapper.Metadata.STAND_LOAD_TIME;
  16. import static nl.b3p.brmo.bgt.loader.Utils.getMessageFormattedString;

  18. import java.io.FileInputStream;
  19. import java.io.IOException;
  20. import java.io.InputStream;
  21. import java.net.CookieManager;
  22. import java.net.URI;
  23. import java.net.http.HttpClient;
  24. import java.net.http.HttpRequest;
  25. import java.net.http.HttpResponse;
  26. import java.nio.file.Files;
  27. import java.nio.file.Path;
  28. import java.sql.PreparedStatement;
  29. import java.text.SimpleDateFormat;
  30. import java.time.LocalDate;
  31. import java.util.Date;
  32. import java.util.HashMap;
  33. import java.util.HashSet;
  34. import java.util.Map;
  35. import java.util.Set;
  36. import java.util.regex.Matcher;
  37. import java.util.regex.Pattern;
  38. import nl.b3p.brmo.bag2.loader.BAG2Database;
  39. import nl.b3p.brmo.bag2.loader.BAG2GMLMutatieGroepStream;
  40. import nl.b3p.brmo.bag2.loader.BAG2LoaderUtils;
  41. import nl.b3p.brmo.bag2.loader.BAG2ProgressReporter;
  42. import nl.b3p.brmo.bag2.schema.BAG2ObjectTableWriter;
  43. import nl.b3p.brmo.bag2.schema.BAG2ObjectType;
  44. import nl.b3p.brmo.bag2.schema.BAG2Schema;
  45. import nl.b3p.brmo.bag2.schema.BAG2SchemaMapper;
  46. import nl.b3p.brmo.schema.ObjectType;
  47. import nl.b3p.brmo.sql.GeometryHandlingPreparedStatementBatch;
  48. import nl.b3p.brmo.sql.LoggingQueryRunner;
  49. import nl.b3p.brmo.sql.QueryBatch;
  50. import nl.b3p.brmo.sql.dialect.OracleDialect;
  51. import nl.b3p.brmo.util.ResumingInputStream;
  52. import nl.b3p.brmo.util.http.HttpClientWrapper;
  53. import nl.b3p.brmo.util.http.HttpStartRangeInputStreamProvider;
  54. import nl.b3p.brmo.util.http.wrapper.Java11HttpClientWrapper;
  55. import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
  56. import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
  57. import org.apache.commons.io.input.CloseShieldInputStream;
  58. import org.apache.commons.lang3.tuple.Pair;
  59. import org.apache.commons.logging.Log;
  60. import org.apache.commons.logging.LogFactory;
  61. import org.apache.log4j.PropertyConfigurator;
  62. import org.geotools.util.logging.Logging;
  63. import org.locationtech.jts.geom.Geometry;
  64. import org.locationtech.jts.io.WKTReader;
  65. import picocli.CommandLine;
  66. import picocli.CommandLine.Command;
  67. import picocli.CommandLine.ExitCode;
  68. import picocli.CommandLine.IVersionProvider;
  69. import picocli.CommandLine.Mixin;
  70. import picocli.CommandLine.Option;
  71. import picocli.CommandLine.Parameters;

  73. @Command(
  74.     name = "bag2-loader",
  75.     mixinStandardHelpOptions = true,
  76.     versionProvider = BAG2LoaderMain.class,
  77.     resourceBundle = BAG2LoaderUtils.BUNDLE_NAME,
  78.     subcommands = {BAG2MutatiesCommand.class})
  79. public class BAG2LoaderMain implements IVersionProvider {
  80.   private static Log log;

  82.   /* zodat we een JNDI database kunnen gebruiken */
  83.   private BAG2Database bag2Database = null;

  85.   private Set<BAG2ObjectType> objectTypesWithSchemaCreated = new HashSet<>();

  87.   private Map<BAG2ObjectType, Set<Pair<Object, Object>>> keysPerObjectType = new HashMap<>();

  89.   /**
  90.    * init logging.
  91.    *
  92.    * @param standAlone set to {@code false} when using in a preconfigured environment, eg. calling
  93.    *     methods from a servlet, use {@code true} for commandline usage.
  94.    */
  95.   public static void configureLogging(boolean standAlone) {
  96.     if (standAlone) {
  97.       PropertyConfigurator.configure(
  98.           BAG2LoaderMain.class.getResourceAsStream("/bag2-loader-cli-log4j.properties"));
  99.       log = LogFactory.getLog(BAG2LoaderMain.class);
  100.       try {
  101.         Logging.ALL.setLoggerFactory("org.geotools.util.logging.Log4JLoggerFactory");
  102.       } catch (ClassNotFoundException ignored) {
  103.       }
  104.     } else {
  105.       log = LogFactory.getLog(BAG2LoaderMain.class);
  106.     }
  107.   }

  109.   public static void main(String... args) {
  110.     configureLogging(true);

  112.     CommandLine cmd = new CommandLine(new BAG2LoaderMain()).setUsageHelpAutoWidth(true);
  113.     System.exit(cmd.execute(args));
  114.   }

  116.   @Override
  117.   public String[] getVersion() {
  118.     return new String[] {BAG2LoaderUtils.getLoaderVersion(), BAG2LoaderUtils.getUserAgent()};
  119.   }

  121.   @Command(name = "load", sortOptions = false)
  122.   public int load(
  123.       @Mixin BAG2DatabaseOptions dbOptions,
  124.       @Mixin BAG2LoadOptions loadOptions,
  125.       @Mixin BAG2ProgressOptions progressOptions,
  126.       @Parameters(paramLabel = "<file>") String[] filenames,
  127.       @Option(
  128.               names = {"-h", "--help"},
  129.               usageHelp = true)
  130.           boolean showHelp)
  131.       throws Exception {

  133.     log.info(BAG2LoaderUtils.getUserAgent());

  135.     try (BAG2Database db = getBAG2Database(dbOptions)) {
  136.       BAG2ProgressReporter progressReporter =
  137.           progressOptions.isConsoleProgressEnabled()
  138.               ? new BAG2ConsoleProgressReporter()
  139.               : new BAG2ProgressReporter();

  141.       loadFiles(db, dbOptions, loadOptions, progressReporter, filenames, null);
  142.       return ExitCode.OK;
  143.     }
  144.   }

  146.   @Command(name = "apply-geo-filter", sortOptions = false)
  147.   public int applyGeoFilterCommand(
  148.       @Mixin BAG2DatabaseOptions dbOptions,
  149.       @CommandLine.Option(names = "--geo-filter", paramLabel = "<wkt>") String geoFilter)
  150.       throws Exception {
  151.     log.info(BAG2LoaderUtils.getUserAgent());

  153.     try (BAG2Database db = getBAG2Database(dbOptions)) {
  154.       applyGeoFilter(db, new BAG2LoadOptions().setGeoFilter(geoFilter));
  155.       return ExitCode.OK;
  156.     }
  157.   }

  159.   public BAG2Database getBAG2Database(BAG2DatabaseOptions dbOptions) throws ClassNotFoundException {
  160.     if (bag2Database == null) {
  161.       bag2Database = new BAG2Database(dbOptions);
  162.     }
  163.     return bag2Database;
  164.   }

  166.   public void setBag2Database(BAG2Database bag2Database) {
  167.     this.bag2Database = bag2Database;
  168.   }

  170.   public void loadFiles(
  171.       BAG2Database db,
  172.       BAG2DatabaseOptions dbOptions,
  173.       BAG2LoadOptions loadOptions,
  174.       BAG2ProgressReporter progressReporter,
  175.       String[] filenames,
  176.       CookieManager cookieManager)
  177.       throws Exception {

  179.     if (filenames.length == 1 && Files.isDirectory(Path.of(filenames[0]))) {
  180.       log.info("Directory opgegeven, kijken naar toepasbare mutaties...");
  181.       filenames =
  182.           Files.list(Path.of(filenames[0]))
  183.               .filter(p -> !Files.isDirectory(p) && p.getFileName().toString().endsWith(".zip"))
  184.               .map(p -> p.toAbsolutePath().toString())
  185.               .toArray(String[]::new);
  186.       if (filenames.length == 0) {
  187.         log.info("Geen ZIP bestanden gevonden, niets te doen");
  188.       } else {
  189.         applyMutaties(db, dbOptions, loadOptions, progressReporter, filenames, null);
  190.       }
  191.       return;
  192.     }

  194.     BAG2LoaderUtils.BAGExtractSelectie bagExtractLevering =
  195.         BAG2LoaderUtils.getBAGExtractSelectieFromZip(filenames[0]);

  197.     if (bagExtractLevering.isStand()) {
  198.       if (bagExtractLevering.isGebiedNLD()) {
  199.         if (filenames.length > 1) {
  200.           throw new IllegalArgumentException(
  201.               "Inladen stand heel Nederland: teveel bestanden opgegeven");
  202.         }
  203.       } else {
  204.         // Verify all filenames are gemeentestanden
  205.         Set<String> gemeenteCodes = new HashSet<>();
  206.         for (int i = 1; i < filenames.length; i++) {
  207.           BAG2LoaderUtils.BAGExtractSelectie nextBagExtractLevering =
  208.               BAG2LoaderUtils.getBAGExtractSelectieFromZip(filenames[i]);

  210.           if (!nextBagExtractLevering.isStand() || nextBagExtractLevering.isGebiedNLD()) {
  211.             throw new IllegalArgumentException(
  212.                 "Inladen stand gemeentes, ongeldig bestand opgegeven (geen gemeentestand): "
  213.                     + filenames[i]);
  214.           }
  215.           Set<String> nextBagExtractLeveringGemeenteCodes =
  216.               nextBagExtractLevering.getGemeenteCodes();
  217.           if (gemeenteCodes.stream().anyMatch(nextBagExtractLeveringGemeenteCodes::contains)) {
  218.             throw new IllegalArgumentException(
  219.                 "Inladen stand gemeentes, dubbele gemeentecode in bestand: " + filenames[i]);
  220.           }
  221.           gemeenteCodes.addAll(nextBagExtractLeveringGemeenteCodes);
  222.         }
  223.       }

  225.       new LoggingQueryRunner().update(db.getConnection(), "create sequence objectid_seq");

  227.       loadStandFiles(db, dbOptions, loadOptions, progressReporter, filenames, cookieManager);
  228.     } else {
  229.       // Process mutaties while ignoring files not applicable
  230.       applyMutaties(db, dbOptions, loadOptions, progressReporter, filenames, null);
  231.     }
  232.   }

  234.   /**
  235.    * Only called after list of files have been checked to only have been entire NL stand or unique
  236.    * gemeente standen.
  237.    */
  238.   private void loadStandFiles(
  239.       BAG2Database db,
  240.       BAG2DatabaseOptions dbOptions,
  241.       BAG2LoadOptions loadOptions,
  242.       BAG2ProgressReporter progressReporter,
  243.       String[] filenames,
  244.       CookieManager cookieManager)
  245.       throws Exception {
  246.     try {
  247.       // When loading multiple standen (for gemeentes), set ignore duplicates so the seen
  248.       // object keys are kept in
  249.       // memory so duplicates can be ignored. Don't keep keys in memory for entire NL stand.
  250.       loadOptions.setIgnoreDuplicates(filenames.length > 1);

  252.       // Multiple gemeentes can also be provided in a single ZIP file, check the
  253.       // Leveringsdocument whether that is
  254.       // the case
  255.       if (filenames.length == 1) {
  256.         BAG2LoaderUtils.BAGExtractSelectie levering =
  257.             BAG2LoaderUtils.getBAGExtractSelectieFromZip(filenames[0]);
  258.         if (!levering.isGebiedNLD()) {
  259.           loadOptions.setIgnoreDuplicates(levering.getGemeenteCodes().size() > 1);
  260.         }
  261.       }

  263.       BAG2GMLMutatieGroepStream.BagInfo bagInfo = null;
  264.       String lastFilename = null;

  266.       // Keep track of which gemeentes are loaded so the correct mutations can be processed
  267.       // later
  268.       Set<String> gemeenteIdentificaties = new HashSet<>();

  270.       for (String filename : filenames) {
  271.         BAG2GMLMutatieGroepStream.BagInfo latestBagInfo =
  272.             loadBAG2ExtractFromURLorFile(db, loadOptions, dbOptions, progressReporter, filename);
  273.         if (bagInfo != null) {
  274.           // For gemeentes the BagInfo must be the same so the standen are of the same
  275.           // date
  276.           if (!latestBagInfo.equalsExceptGemeenteIdentificaties(bagInfo)) {
  277.             throw new IllegalArgumentException(
  278.                 String.format(
  279.                     "Incompatible BagInfo for file \"%s\" (%s) compared to last file \"%s\" (%s)",
  280.                     filename, latestBagInfo, lastFilename, bagInfo));
  281.           }
  282.         }
  283.         bagInfo = latestBagInfo;

  285.         // For NL stand this will be "9999"
  286.         gemeenteIdentificaties.addAll(bagInfo.getGemeenteIdentificaties());
  287.         lastFilename = filename;
  288.       }
  289.       if (bagInfo != null) {
  290.         // TODO: when loading gemeente without rare objects such as
  291.         // ligplaatsen/standplaatsen, table will not be created
  292.         // and a future change with such an object will fail. Should create entire schema
  293.         // up-front instead of when first
  294.         // encountering object type
  295.         createKeysAndIndexes(db, loadOptions, dbOptions, progressReporter);

  297.         updateMetadata(
  298.             db, loadOptions, true, gemeenteIdentificaties, bagInfo.getStandTechnischeDatum());
  299.       }
  300.       db.getConnection().commit();

  302.       applyGeoFilter(db, loadOptions);
  303.     } finally {
  304.       progressReporter.reportTotalSummary();
  305.     }
  306.   }

  308.   public void applyMutaties(
  309.       BAG2Database db,
  310.       BAG2DatabaseOptions dbOptions,
  311.       BAG2LoadOptions loadOptions,
  312.       BAG2ProgressReporter progressReporter,
  313.       String[] urls,
  314.       CookieManager cookieManager)
  315.       throws Exception {
  316.     if (urls.length == 0) {
  317.       return;
  318.     }
  319.     BAG2LoaderUtils.BAGExtractSelectie bagExtractLevering =
  320.         BAG2LoaderUtils.getBAGExtractSelectieFromZip(urls[0]);
  321.     if (bagExtractLevering.isGebiedNLD()) {
  322.       applyNLMutaties(db, dbOptions, loadOptions, progressReporter, urls, cookieManager);
  323.     } else {
  324.       applyGemeenteMutaties(db, dbOptions, loadOptions, progressReporter, urls, cookieManager);
  325.     }
  326.   }

  328.   private void applyGemeenteMutaties(
  329.       BAG2Database db,
  330.       BAG2DatabaseOptions dbOptions,
  331.       BAG2LoadOptions loadOptions,
  332.       BAG2ProgressReporter progressReporter,
  333.       String[] urls,
  334.       CookieManager cookieManager)
  335.       throws Exception {
  336.     LocalDate currentTechnischeDatum = db.getCurrentTechnischeDatum();
  337.     Set<String> gemeenteCodes = db.getGemeenteCodes();

  339.     Set<Integer> applicableMutatieIndexes;
  340.     do {
  341.       applicableMutatieIndexes = new HashSet<>();

  343.       Set<String> missingGemeentes = new HashSet<>(gemeenteCodes);
  344.       for (int i = 0; i < urls.length; i++) {
  345.         BAG2LoaderUtils.BAGExtractSelectie bagExtractLevering =
  346.             BAG2LoaderUtils.getBAGExtractSelectieFromZip(urls[i]);

  348.         if (!bagExtractLevering.isGebiedNLD()
  349.             && !bagExtractLevering.isStand()
  350.             && bagExtractLevering.getMutatiesFrom().equals(currentTechnischeDatum)) {

  352.           for (String gemeenteCode : bagExtractLevering.getGemeenteCodes()) {
  353.             if (gemeenteCodes.contains(gemeenteCode)) {
  354.               applicableMutatieIndexes.add(i);
  355.               missingGemeentes.remove(gemeenteCode);
  356.             }
  357.           }
  358.         }
  359.       }

  361.       if (applicableMutatieIndexes.isEmpty()) {
  362.         log.info(
  363.             String.format(
  364.                 "Geen nieuw toe te passen gemeentemutatiebestanden gevonden voor huidige stand technische datum %s, klaar",
  365.                 currentTechnischeDatum));
  366.         break;
  367.       }

  369.       // Check whether applicable mutaties are available for all gemeentecodes because they
  370.       // need to be processed
  371.       // at the same time to ignore duplicates
  372.       if (!missingGemeentes.isEmpty()) {
  373.         throw new IllegalArgumentException(
  374.             String.format(
  375.                 "Kan geen gemeente mutaties toepassen voor gemeentes %s vanaf stand technische datum %s, in opgegeven mutatiebestanden ontbreken gemeentecodes %s",
  376.                 gemeenteCodes, currentTechnischeDatum, missingGemeentes));
  377.       }

  379.       log.info(
  380.           String.format(
  381.               "Toepassen gemeentemutaties voor %d gemeentes vanaf stand technische datum %s...",
  382.               gemeenteCodes.size(), currentTechnischeDatum));

  384.       BAG2GMLMutatieGroepStream.BagInfo bagInfo = null;

  386.       loadOptions.setIgnoreDuplicates(gemeenteCodes.size() > 1);
  387.       for (int index : applicableMutatieIndexes) {
  388.         bagInfo =
  389.             loadBAG2ExtractFromURLorFile(
  390.                 db, loadOptions, dbOptions, progressReporter, urls[index], cookieManager);
  391.       }
  392.       currentTechnischeDatum =
  393.           new java.sql.Date(bagInfo.getStandTechnischeDatum().getTime()).toLocalDate();
  394.       updateMetadata(db, loadOptions, false, null, bagInfo.getStandTechnischeDatum());
  395.       db.getConnection().commit();
  396.       // Duplicates need only be checked for mutaties for a single from date, clear cache to
  397.       // reduce memory usage
  398.       clearDuplicatesCache();
  399.       log.info("Mutaties verwerkt, huidige stand technische datum: " + currentTechnischeDatum);

  401.     } while (true);
  402.     applyGeoFilter(db, loadOptions);
  403.   }

  405.   private void applyNLMutaties(
  406.       BAG2Database db,
  407.       BAG2DatabaseOptions dbOptions,
  408.       BAG2LoadOptions loadOptions,
  409.       BAG2ProgressReporter progressReporter,
  410.       String[] urls,
  411.       CookieManager cookieManager)
  412.       throws Exception {
  413.     LocalDate currentTechnischeDatum = db.getCurrentTechnischeDatum();
  414.     do {
  415.       String applicatieMutatieURL = null;

  417.       for (String url : urls) {
  418.         BAG2LoaderUtils.BAGExtractSelectie bagExtractSelectie =
  419.             BAG2LoaderUtils.getBAGExtractSelectieFromZip(url);

  421.         if (bagExtractSelectie.isGebiedNLD()
  422.             && !bagExtractSelectie.isStand()
  423.             && bagExtractSelectie.getMutatiesFrom().equals(currentTechnischeDatum)) {
  424.           applicatieMutatieURL = url;
  425.         }
  426.       }

  428.       if (applicatieMutatieURL == null) {
  429.         log.info(
  430.             String.format(
  431.                 "Geen nieuw toe te passen mutatiebestanden gevonden voor huidige stand technische datum %s, klaar",
  432.                 currentTechnischeDatum));
  433.         break;
  434.       }

  436.       log.info(
  437.           String.format(
  438.               "Toepassen mutaties vanaf stand technische datum %s...", currentTechnischeDatum));

  440.       BAG2GMLMutatieGroepStream.BagInfo bagInfo =
  441.           loadBAG2ExtractFromURLorFile(
  442.               db, loadOptions, dbOptions, progressReporter, applicatieMutatieURL, cookieManager);
  443.       currentTechnischeDatum =
  444.           new java.sql.Date(bagInfo.getStandTechnischeDatum().getTime()).toLocalDate();
  445.       updateMetadata(db, loadOptions, false, null, bagInfo.getStandTechnischeDatum());
  446.       db.getConnection().commit();
  447.       log.info("Mutaties verwerkt, huidige stand technische datum: " + currentTechnischeDatum);
  448.     } while (true);
  449.     applyGeoFilter(db, loadOptions);
  450.   }

  452.   private static final int SRID = 28992;

  454.   public void applyGeoFilter(BAG2Database db, BAG2LoadOptions loadOptions) throws Exception {

  456.     String filterMetadata;

  458.     if (loadOptions.getGeoFilter() != null) {
  459.       filterMetadata = loadOptions.getGeoFilter();
  460.       try {
  461.         new WKTReader().read(filterMetadata);
  462.       } catch (Exception e) {
  463.         log.error("Ongeldige WKT gespecificeerd voor geometrie-filter", e);
  464.         return;
  465.       }
  466.       db.setMetadataValue(FILTER_GEOMETRIE, filterMetadata);
  467.     } else {
  468.       filterMetadata = db.getMetadata(FILTER_GEOMETRIE);
  469.     }

  471.     if (filterMetadata == null) {
  472.       return;
  473.     }
  474.     Geometry geometry;
  475.     try {
  476.       geometry = new WKTReader().read(filterMetadata);
  477.       geometry.setSRID(SRID);
  478.     } catch (Exception e) {
  479.       log.error(
  480.           String.format(
  481.               "Geometrie-filter niet toegepast, fout bij parsen %s als WKT, waarde: \"%s\"",
  482.               FILTER_GEOMETRIE, filterMetadata),
  483.           e);
  484.       return;
  485.     }
  486.     log.info("Verwijderen records die niet binnen geometrie-filter vallen...");

  488.     BAG2SchemaMapper schemaMapper = BAG2SchemaMapper.getInstance();
  489.     BAG2Schema bag2Schema = BAG2Schema.getInstance();

  491.     db.getConnection().setAutoCommit(false);

  493.     String[] objectTypes =
  494.         new String[] {"Ligplaats", "Standplaats", "Pand", "Verblijfsobject", "Woonplaats"};

  496.     for (String objectTypeName : objectTypes) {
  497.       ObjectType objectType = bag2Schema.getObjectTypeByName(objectTypeName);
  498.       String tableName = schemaMapper.getTableNameForObjectType(objectType, "");
  499.       String geoCondition =
  500.           db.getDialect() instanceof OracleDialect
  501.               ? "st_intersects(?, geometrie) = 'FALSE'"
  502.               : "not st_intersects(geometrie, ?)"; // Also works for Oracle 23c, not < 23c
  503.       String sql =
  504.           String.format(
  505.               """
  506.                 delete from %s
  507.                 where identificatie in (
  508.                   select identificatie from %s
  509.                   where eindgeldigheid is null and tijdstipinactief is null
  510.                   and %s
  511.                 )""",
  512.               tableName, tableName, geoCondition);
  513.       try (QueryBatch queryBatch =
  514.           new GeometryHandlingPreparedStatementBatch(
  515.               db.getConnection(), sql, 1, db.getDialect(), new Boolean[] {true}, false)) {
  516.         log.info("Verwijderen van records voor " + objectType.getName());
  517.         queryBatch.addBatch(new Object[] {geometry});
  518.       }
  519.     }

  521.     // Don't use getObjectTypeByName(), because heeftAlsNevenadres join tables need to be directly
  522.     // in sql literal anyway
  523.     String sql =
  524.         """
  525.         delete from nummeraanduiding n
  526.         where not exists (select 1 from verblijfsobject v where v.heeftalshoofdadres = n.identificatie)
  527.         and not exists (select 1 from verblijfsobject_nevenadres vn where vn.heeftalsnevenadres = n.identificatie)
  528.         and not exists (select 1 from ligplaats l where l.heeftalshoofdadres = n.identificatie)
  529.         and not exists (select 1 from ligplaats_nevenadres ln where ln.heeftalsnevenadres = n.identificatie)
  530.         and not exists (select 1 from standplaats s where s.heeftalshoofdadres = n.identificatie)
  531.         and not exists (select 1 from standplaats_nevenadres sn where sn.heeftalsnevenadres = n.identificatie)
  532.         """;
  533.     try (PreparedStatement ps = db.getConnection().prepareStatement(sql)) {
  534.       log.info("Verwijderen niet-gerefereerde nummeraanduiding-records");
  535.       ps.executeUpdate();
  536.     }
  537.     sql =
  538.         """
  539.         delete from openbareruimte o where not exists (select 1 from nummeraanduiding where ligtaan = o.identificatie)
  540.         """;
  541.     try (PreparedStatement ps = db.getConnection().prepareStatement(sql)) {
  542.       log.info("Verwijderen niet-gerefereerde openbareruimte-records");
  543.       ps.executeUpdate();
  544.     }
  545.     db.getConnection().commit();
  546.     log.info("Klaar met verwijderen records buiten geometrie-filter");
  547.   }

  549.   private void createKeysAndIndexes(
  550.       BAG2Database db,
  551.       BAG2LoadOptions loadOptions,
  552.       BAG2DatabaseOptions databaseOptions,
  553.       BAG2ProgressReporter progressReporter)
  554.       throws Exception {
  555.     BAG2ObjectTableWriter writer = db.createObjectTableWriter(loadOptions, databaseOptions);
  556.     writer.setProgressUpdater(progressReporter);
  557.     for (BAG2ObjectType objectType : objectTypesWithSchemaCreated) {
  558.       writer.createKeys(objectType); // BAG2 writer is always a single ObjectType unlike BGT
  559.       writer.createIndexes(objectType);
  560.     }
  561.   }

  563.   private BAG2GMLMutatieGroepStream.BagInfo loadBAG2ExtractFromURLorFile(
  564.       BAG2Database db,
  565.       BAG2LoadOptions loadOptions,
  566.       BAG2DatabaseOptions dbOptions,
  567.       BAG2ProgressReporter progressReporter,
  568.       String url)
  569.       throws Exception {
  570.     return loadBAG2ExtractFromURLorFile(db, loadOptions, dbOptions, progressReporter, url, null);
  571.   }

  573.   private BAG2GMLMutatieGroepStream.BagInfo loadBAG2ExtractFromURLorFile(
  574.       BAG2Database db,
  575.       BAG2LoadOptions loadOptions,
  576.       BAG2DatabaseOptions dbOptions,
  577.       BAG2ProgressReporter progressReporter,
  578.       String url,
  579.       CookieManager cookieManager)
  580.       throws Exception {
  581.     HttpClientWrapper<HttpRequest.Builder, HttpResponse<InputStream>> httpClientWrapper =
  582.         cookieManager == null
  583.             ? new Java11HttpClientWrapper()
  584.             : new Java11HttpClientWrapper(HttpClient.newBuilder().cookieHandler(cookieManager));

  586.     if (url.startsWith("http://") || url.startsWith("https://")) {
  587.       try (InputStream in =
  588.           new ResumingInputStream(
  589.               new HttpStartRangeInputStreamProvider(URI.create(url), httpClientWrapper))) {
  590.         return loadBAG2ExtractFromStream(db, loadOptions, dbOptions, progressReporter, url, in);
  591.       }
  592.     }
  593.     if (url.endsWith(".zip")) {
  594.       try (InputStream in = new FileInputStream(url)) {
  595.         return loadBAG2ExtractFromStream(db, loadOptions, dbOptions, progressReporter, url, in);
  596.       }
  597.     }

  599.     throw new IllegalArgumentException(getMessageFormattedString("load.invalid_file", url));
  600.   }

  602.   private BAG2GMLMutatieGroepStream.BagInfo loadBAG2ExtractFromStream(
  603.       BAG2Database db,
  604.       BAG2LoadOptions loadOptions,
  605.       BAG2DatabaseOptions dbOptions,
  606.       BAG2ProgressReporter progressReporter,
  607.       String name,
  608.       InputStream input)
  609.       throws Exception {
  610.     BAG2GMLMutatieGroepStream.BagInfo bagInfo = null;
  611.     Set<String> gemeenteIdentificaties = new HashSet<>();
  612.     try (ZipArchiveInputStream zip = new ZipArchiveInputStream(input)) {
  613.       ZipArchiveEntry entry = zip.getNextZipEntry();
  614.       while (entry != null) {
  615.         if (entry.getName().matches("[0-9]{4}(STA|VBO|OPR|NUM|LIG|PND|WPL).*\\.xml")) {
  616.           // Load extracted zipfile
  617.           bagInfo =
  618.               loadXmlEntriesFromZipFile(
  619.                   db, loadOptions, dbOptions, progressReporter, name, zip, entry);
  620.           break;
  621.         }

  623.         if (entry.getName().matches("[0-9]{4}GEM[0-9]{8}\\.zip")) {
  624.           bagInfo =
  625.               loadBAG2ExtractFromStream(
  626.                   db,
  627.                   loadOptions,
  628.                   dbOptions,
  629.                   progressReporter,
  630.                   name,
  631.                   CloseShieldInputStream.wrap(zip));
  632.         }

  634.         // Process single and double-nested ZIP files

  636.         if (entry.getName().matches("[0-9]{4}(STA|VBO|OPR|NUM|LIG|PND|WPL).*\\.zip")
  637.             || entry.getName().matches("[0-9]{4}MUT[0-9]{8}-[0-9]{8}\\.zip")) {
  638.           ZipArchiveInputStream nestedZip = new ZipArchiveInputStream(zip);
  639.           bagInfo =
  640.               loadXmlEntriesFromZipFile(
  641.                   db,
  642.                   loadOptions,
  643.                   dbOptions,
  644.                   progressReporter,
  645.                   entry.getName(),
  646.                   nestedZip,
  647.                   nestedZip.getNextZipEntry());
  648.         }

  650.         if (entry.getName().matches("[0-9]{4}Inactief.*\\.zip")) {
  651.           ZipArchiveInputStream nestedZip = new ZipArchiveInputStream(zip);
  652.           ZipArchiveEntry nestedEntry = nestedZip.getNextZipEntry();
  653.           while (nestedEntry != null) {
  654.             if (nestedEntry.getName().matches("[0-9]{4}IA.*\\.zip")) {
  655.               ZipArchiveInputStream moreNestedZip = new ZipArchiveInputStream(nestedZip);
  656.               bagInfo =
  657.                   loadXmlEntriesFromZipFile(
  658.                       db,
  659.                       loadOptions,
  660.                       dbOptions,
  661.                       progressReporter,
  662.                       nestedEntry.getName(),
  663.                       moreNestedZip,
  664.                       moreNestedZip.getNextZipEntry());
  665.             }
  666.             nestedEntry = nestedZip.getNextZipEntry();
  667.           }
  668.         }

  670.         if (bagInfo != null) {
  671.           gemeenteIdentificaties.addAll(bagInfo.getGemeenteIdentificaties());
  672.         }

  674.         try {
  675.           entry = zip.getNextZipEntry();
  676.         } catch (IOException e) {
  677.           // Reading the ZIP from HTTP may give this error, but it is a normal end...
  678.           if ("Truncated ZIP file".equals(e.getMessage())) {
  679.             break;
  680.           }
  681.         }
  682.       }
  683.     }
  684.     if (bagInfo != null) {
  685.       // Update BagInfo with all seen gemeenteIdentificaties in case we processed a ZIP with
  686.       // multiple gemeentestanden
  687.       bagInfo.setGemeenteIdentificaties(gemeenteIdentificaties);
  688.     }
  689.     return bagInfo;
  690.   }

  692.   private static BAG2ObjectType getObjectTypeFromFilename(String filename) {
  693.     Matcher m =
  694.         Pattern.compile(".*[0-9]{4}(IA)?(MUT|STA|VBO|OPR|NUM|LIG|PND|WPL).*\\.(xml|zip)")
  695.             .matcher(filename);
  696.     if (!m.matches()) {
  697.       throw new IllegalArgumentException("Invalid BAG2 filename: " + filename);
  698.     }
  699.     String objectTypeName = null;
  700.     switch (m.group(2)) {
  701.       case "MUT":
  702.         break;
  703.       case "STA":
  704.         objectTypeName = "Standplaats";
  705.         break;
  706.       case "OPR":
  707.         objectTypeName = "OpenbareRuimte";
  708.         break;
  709.       case "VBO":
  710.         objectTypeName = "Verblijfsobject";
  711.         break;
  712.       case "NUM":
  713.         objectTypeName = "Nummeraanduiding";
  714.         break;
  715.       case "LIG":
  716.         objectTypeName = "Ligplaats";
  717.         break;
  718.       case "PND":
  719.         objectTypeName = "Pand";
  720.         break;
  721.       case "WPL":
  722.         objectTypeName = "Woonplaats";
  723.         break;
  724.     }
  725.     if (objectTypeName == null) {
  726.       return null;
  727.     } else {
  728.       return BAG2Schema.getInstance().getObjectTypeByName(objectTypeName);
  729.     }
  730.   }

  732.   private void updateMetadata(
  733.       BAG2Database db,
  734.       BAG2LoadOptions loadOptions,
  735.       boolean stand,
  736.       Set<String> gemeenteIdentificaties,
  737.       Date standTechnischeDatum)
  738.       throws Exception {
  739.     // Check if metadata table already exists. For PostgreSQL we can use the metadata table in
  740.     // the public schema
  741.     if (!db.getDialect().tableExists(db.getConnection(), METADATA_TABLE_NAME)) {
  742.       // Create a new metadata table, for Oracle as BAG is in separate schema, for PostgreSQL
  743.       // if loading BAG
  744.       // into a non-brmo RSGB database
  745.       db.createMetadataTable(loadOptions);
  746.     }

  748.     db.setMetadataValue(
  749.         BAG2SchemaMapper.Metadata.LOADER_VERSION, BAG2LoaderUtils.getLoaderVersion());
  750.     SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd");
  751.     if (stand) {
  752.       db.setMetadataValue(
  753.           STAND_LOAD_TIME, new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()));
  754.       db.setMetadataValue(STAND_LOAD_TECHNISCHE_DATUM, df.format(standTechnischeDatum));
  755.       db.setMetadataValue(GEMEENTE_CODES, String.join(",", gemeenteIdentificaties));
  756.     }
  757.     db.setMetadataValue(CURRENT_TECHNISCHE_DATUM, df.format(standTechnischeDatum));
  758.   }

  760.   private void clearDuplicatesCache() {
  761.     keysPerObjectType = new HashMap<>();
  762.   }

  764.   private BAG2GMLMutatieGroepStream.BagInfo loadXmlEntriesFromZipFile(
  765.       BAG2Database db,
  766.       BAG2LoadOptions loadOptions,
  767.       BAG2DatabaseOptions databaseOptions,
  768.       BAG2ProgressReporter progressReporter,
  769.       String name,
  770.       ZipArchiveInputStream zip,
  771.       ZipArchiveEntry entry)
  772.       throws Exception {
  773.     BAG2ObjectType objectType = getObjectTypeFromFilename(name);
  774.     // objectType is null for mutaties, which contain mixed object types instead of a single
  775.     // object type with stand
  776.     boolean schemaCreated = objectType == null || objectTypesWithSchemaCreated.contains(objectType);
  777.     BAG2ObjectTableWriter writer = db.createObjectTableWriter(loadOptions, databaseOptions);
  778.     writer.setProgressUpdater(progressReporter);
  779.     writer.setCreateSchema(!schemaCreated);
  780.     writer.setCreateKeysAndIndexes(false);
  781.     writer.setKeysPerObjectType(keysPerObjectType);
  782.     writer.start(); // sets InitialLoad to true
  783.     writer
  784.         .getProgress()
  785.         .setInitialLoad(!schemaCreated); // For a COPY in transaction, table must be created or
  786.     // truncated in it
  787.     if (objectType == null) {
  788.       // When processing mutaties, set batch size to 1 so all mutaties are processed
  789.       // sequentially and can not
  790.       // conflict with deleting and inserting of old/new versions
  791.       writer.setBatchSize(1);
  792.       // Disable multithreading so deletion of previous versions and new inserts are processed
  793.       // sequentially
  794.       writer.setMultithreading(false);
  795.     }
  796.     progressReporter.startNewFile(name);
  797.     try {
  798.       while (entry != null) {
  799.         progressReporter.startNextSplitFile(entry.getName());
  800.         writer.write(CloseShieldInputStream.wrap(zip));
  801.         if (loadOptions.getMaxObjects() != null
  802.             && writer.getProgress().getObjectCount() == loadOptions.getMaxObjects()) {
  803.           break;
  804.         }
  805.         entry = zip.getNextZipEntry();
  806.       }
  807.       writer.complete();

  809.       if (writer.getProgress().getObjectCount() > 0 && objectType != null) {
  810.         objectTypesWithSchemaCreated.add(objectType);
  811.       }

  813.       return writer.getProgress().getMutatieInfo();
  814.     } catch (Exception e) {
  815.       writer.abortWorkerThread();
  816.       throw e;
  817.     }
  818.   }
  819. }
Created with JaCoCo 0.8.12.202403310830