diff --git a/amp/Dockerfile b/amp/Dockerfile index 06687fe9ba2..1654e664b33 100644 --- a/amp/Dockerfile +++ b/amp/Dockerfile @@ -178,6 +178,10 @@ COPY --from=compile-amp-filter /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-fi COPY --from=compile-amp-settings /tmp/amp/TEMPLATE/ampTemplate/node_modules/amp-settings ../ampTemplate/node_modules/amp-settings # Copy package files for dependency installation COPY TEMPLATE/reampv2/package*.json ./ +COPY TEMPLATE/reampv2/packages/container/package*.json ./packages/container/ +COPY TEMPLATE/reampv2/packages/ampoffline/package*.json ./packages/ampoffline/ +COPY TEMPLATE/reampv2/packages/reampv2-app/package*.json ./packages/reampv2-app/ +COPY TEMPLATE/reampv2/packages/user-manager/package*.json ./packages/user-manager/ RUN --mount=type=cache,target=/root/.npm \ --mount=type=ssh \ npm-install-with-retry.sh diff --git a/amp/src/main/java/org/dgfoundation/amp/ar/ArConstants.java b/amp/src/main/java/org/dgfoundation/amp/ar/ArConstants.java index 43bcfe05720..d7b6ef38acc 100644 --- a/amp/src/main/java/org/dgfoundation/amp/ar/ArConstants.java +++ b/amp/src/main/java/org/dgfoundation/amp/ar/ArConstants.java @@ -387,8 +387,7 @@ public final class ArConstants { // public final static String EXECUTING_AGENCY_PERCENTAGE="Eexecuting Agency Percentage"; - //burkina -// public final static String PROGRAM_PERCENTAGE="Program Percentage"; + public final static String PROGRAM_PERCENTAGE="Program Percentage"; diff --git a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/activity/ActivityInterchangeUtils.java b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/activity/ActivityInterchangeUtils.java index 54fd5c23802..0928a762337 100644 --- a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/activity/ActivityInterchangeUtils.java +++ b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/activity/ActivityInterchangeUtils.java @@ -79,6 +79,7 @@ public static JsonApiResponse importActivity(Map newJson) { if (AmpClientModeHolder.isOfflineClient()) { Workspace team = TeamUtil.getWorkspace(Long.parseLong(newJson.get("team").toString())); diff --git a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/dashboards/services/SectorSchemeDTO.java b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/dashboards/services/SectorSchemeDTO.java index 0f062b3d42a..e143bf1e1a9 100644 --- a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/dashboards/services/SectorSchemeDTO.java +++ b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/dashboards/services/SectorSchemeDTO.java @@ -26,12 +26,12 @@ public class SectorSchemeDTO { private final SectorDTO[] children; public SectorSchemeDTO(AmpSectorScheme scheme, SectorDTO[] children) { - this.ampSecSchemeId = scheme.getAmpSecSchemeId(); - this.secSchemeCode = scheme.getSecSchemeCode(); - this.secSchemeName = scheme.getSecSchemeName(); - this.showInRMFilters = scheme.getShowInRMFilters(); - this.used = scheme.isUsed(); - this.children = children; + this.ampSecSchemeId = scheme != null ? scheme.getAmpSecSchemeId() : null; + this.secSchemeCode = scheme != null ? scheme.getSecSchemeCode() : null; + this.secSchemeName = scheme != null ? scheme.getSecSchemeName() : null; + this.showInRMFilters = Boolean.TRUE.equals(scheme != null ? scheme.getShowInRMFilters() : null); + this.used = scheme != null && scheme.isUsed(); + this.children = children != null ? children : new SectorDTO[0]; } public Long getAmpSecSchemeId() { diff --git a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/indicator/manager/IndicatorManagerService.java b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/indicator/manager/IndicatorManagerService.java index fa6b9c2e4bc..0acd3b49a38 100644 --- a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/indicator/manager/IndicatorManagerService.java +++ b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/indicator/manager/IndicatorManagerService.java @@ -79,6 +79,88 @@ public MEIndicatorDTO getMEIndicatorById(final Long indicatorId) { throw new ApiRuntimeException(BAD_REQUEST, ApiError.toError("Indicator with id " + indicatorId + " not found")); } + public MEIndicatorDTO getMeIndicatorByNameAndProgramName(String name, String programName) { + Session session = PersistenceManager.getSession(); + AmpIndicator indicator; + if (programName==null){ + indicator = (AmpIndicator) session.createCriteria(AmpIndicator.class) + .add(Restrictions.eq("name", name)) + .setMaxResults(1) + .uniqueResult(); + } + else { + indicator = (AmpIndicator) session.createCriteria(AmpIndicator.class) + .add(Restrictions.eq("name", name)) + .createAlias("program", "p") + .add(Restrictions.eq("p.name", programName)) + .setMaxResults(1) + .uniqueResult(); + } + + + + if (indicator != null) { + return new MEIndicatorDTO(indicator); + } + + throw new ApiRuntimeException(BAD_REQUEST, + ApiError.toError("Indicator with name " + name + " and program name " + programName + " not found")); + } + + /** + * Returns the indicator by name and optional program name, or null if not found. + * Use this when you need to look up an indicator without throwing (e.g. data import). + * Tries exact name match first; if not found, looks for an indicator whose name contains + * the given name (e.g. DB "1.2.1 - Number of ... - 1.2.1" matches file "Number of ..."). + */ + public MEIndicatorDTO getMeIndicatorByNameAndProgramNameOptional(String name, String programName) { + if (name == null || name.trim().isEmpty()) { + return null; + } + String trimmedName = name.trim(); + String trimmedProgram = programName != null && !programName.trim().isEmpty() ? programName.trim() : null; + try { + return getMeIndicatorByNameAndProgramName(trimmedName, trimmedProgram); + } catch (ApiRuntimeException e) { + logger.info("getMeIndicatorByNameAndProgramNameOptional: exact match not found, trying substring match for name='" + trimmedName + "' programName='" + trimmedProgram + "'"); + + // exact match not found, try substring match (e.g. DB "1.2.1 - X - 1.2.1" vs file "X") + } + return getMeIndicatorByNameSubstringOptional(trimmedName, trimmedProgram); + } + + /** + * Finds an indicator whose stored name contains the given name (e.g. "1.2.1 - X - 1.2.1" contains "X"). + * Returns null if none or multiple matches (when program not specified), or the match when program is specified. + */ + private MEIndicatorDTO getMeIndicatorByNameSubstringOptional(String name, String programName) { + Session session = PersistenceManager.getSession(); + String escaped = escapeForLike(name); + String pattern = "%" + escaped + "%"; + String hql = "from " + AmpIndicator.class.getName() + " i where i.name like :name escape '\\'"; + if (programName != null) { + hql += " and i.program is not null and i.program.name = :programName"; + } + org.hibernate.query.Query query = session.createQuery(hql); + query.setParameter("name", pattern); + if (programName != null) { + query.setParameter("programName", programName); + } + @SuppressWarnings("unchecked") + List list = (List) query.list(); + if (list == null || list.isEmpty()) { + return null; + } + if (list.size() > 1 && programName == null) { + logger.warn("getMeIndicatorByNameSubstringOptional: multiple indicators contain name substring '" + name + "', returning first"); + } + return new MEIndicatorDTO(list.get(0)); + } + + private static String escapeForLike(String s) { + if (s == null) return null; + return s.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_"); + } public MEIndicatorDTO createMEIndicator(final MEIndicatorDTO indicatorRequest) { Session session = PersistenceManager.getSession(); @@ -167,6 +249,12 @@ private void validateYear(MEIndicatorDTO value) { } private void validateYearRange(String startYear, String endYear, AmpIndicatorGlobalValue value, String error){ + if (value == null) { + return; + } + if (startYear == null || endYear == null) { + return; + } String startInString = "01/01/" + startYear; DateTime dateTime = DateTime.parse(startInString, formatter); diff --git a/amp/src/main/java/org/digijava/kernel/persistence/PersistenceManager.java b/amp/src/main/java/org/digijava/kernel/persistence/PersistenceManager.java index a8ed85d19f5..ab7d307157b 100644 --- a/amp/src/main/java/org/digijava/kernel/persistence/PersistenceManager.java +++ b/amp/src/main/java/org/digijava/kernel/persistence/PersistenceManager.java @@ -881,4 +881,4 @@ public static R doInTransaction(Function fn) { PersistenceManager.closeSession(session); } } -} +} \ No newline at end of file diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java index cc5244d9b79..85d33372438 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java @@ -1,11 +1,34 @@ package org.digijava.module.aim.action.dataimporter; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.opencsv.CSVParser; -import com.opencsv.CSVParserBuilder; -import com.opencsv.CSVReader; -import com.opencsv.CSVReaderBuilder; -import com.opencsv.exceptions.CsvValidationException; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.file.Files; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; @@ -16,11 +39,18 @@ import org.apache.struts.action.ActionForward; import org.apache.struts.action.ActionMapping; import org.digijava.kernel.persistence.PersistenceManager; +import static org.digijava.module.aim.action.dataimporter.ExcelImporter.processExcelFileInBatches; import org.digijava.module.aim.action.dataimporter.dbentity.DataImporterConfig; import org.digijava.module.aim.action.dataimporter.dbentity.DataImporterConfigValues; import org.digijava.module.aim.action.dataimporter.dbentity.ImportStatus; import org.digijava.module.aim.action.dataimporter.dbentity.ImportedFilesRecord; import org.digijava.module.aim.action.dataimporter.util.ImportedFileUtil; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.ConstantsMap; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.isFileContentValid; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.isFileReadable; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.removeMapItem; + +import org.digijava.module.aim.action.dataimporter.util.ImporterConstants; import org.digijava.module.aim.form.DataImporterForm; import org.hibernate.Query; import org.hibernate.Session; @@ -28,22 +58,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.*; -import java.nio.file.Files; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.time.Duration; -import java.time.Instant; -import java.time.LocalDateTime; -import java.util.*; -import java.util.stream.Collectors; - -import static org.digijava.module.aim.action.dataimporter.ExcelImporter.processExcelFileInBatches; -import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.*; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.opencsv.CSVParser; +import com.opencsv.CSVParserBuilder; +import com.opencsv.CSVReader; +import com.opencsv.CSVReaderBuilder; +import com.opencsv.exceptions.CsvValidationException; public class DataImporter extends Action { static Logger logger = LoggerFactory.getLogger(DataImporter.class); @@ -81,59 +101,87 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet logger.info(" this is the action " + request.getParameter("action")); if (request.getParameter("uploadTemplate") != null) { logger.info(" this is the action " + request.getParameter("uploadTemplate")); - Set headersSet = new HashSet<>(); + response.setCharacterEncoding("UTF-8"); + dataImporterForm.getColumnPairs().clear(); if (request.getParameter("fileType") != null) { InputStream fileInputStream = dataImporterForm.getTemplateFile().getInputStream(); - if ((Objects.equals(request.getParameter("fileType"), "excel") || Objects.equals(request.getParameter("fileType"), "csv"))) { - Workbook workbook = new XSSFWorkbook(fileInputStream); - int numberOfSheets = workbook.getNumberOfSheets(); - for (int i = 0; i < numberOfSheets; i++) { - Sheet sheet = workbook.getSheetAt(i); - Row headerRow = sheet.getRow(0); - Iterator cellIterator = headerRow.cellIterator(); - while (cellIterator.hasNext()) { - Cell cell = cellIterator.next(); - headersSet.add(cell.getStringCellValue()); + if (Objects.equals(request.getParameter("fileType"), "excel")) { + // Excel: return sheet names and columns per sheet for template configuration + List sheetNames = new ArrayList<>(); + Map> columnsBySheet = new HashMap<>(); + try (Workbook workbook = new XSSFWorkbook(fileInputStream)) { + int numberOfSheets = workbook.getNumberOfSheets(); + for (int i = 0; i < numberOfSheets; i++) { + Sheet sheet = workbook.getSheetAt(i); + String sheetName = sheet.getSheetName(); + sheetNames.add(sheetName); + List columns = new ArrayList<>(); + Row headerRow = sheet.getRow(0); + if (headerRow != null) { + Iterator cellIterator = headerRow.cellIterator(); + while (cellIterator.hasNext()) { + Cell cell = cellIterator.next(); + String val = cell.getStringCellValue(); + if (val != null && !val.trim().isEmpty()) { + columns.add(val.trim()); + } + } + } + columnsBySheet.put(sheetName, columns.stream().sorted().collect(Collectors.toList())); } - } - workbook.close(); - - + Map jsonResponse = new HashMap<>(); + jsonResponse.put("sheetNames", sheetNames); + jsonResponse.put("columnsBySheet", columnsBySheet); + response.setContentType("application/json"); + new ObjectMapper().writeValue(response.getWriter(), jsonResponse); + } else if (Objects.equals(request.getParameter("fileType"), "csv")) { + Set headersSet = new HashSet<>(); + try (CSVReader reader = new CSVReaderBuilder(new InputStreamReader(fileInputStream)).build()) { + String[] headers = reader.readNext(); + if (headers != null) { + headersSet.addAll(Arrays.asList(headers)); + } + } catch (IOException | CsvValidationException e) { + logger.error("An error occurred during extraction of headers.", e); + } + headersSet = headersSet.stream().sorted().collect(Collectors.toCollection(LinkedHashSet::new)); + StringBuilder headers = new StringBuilder(); + headers.append(" \n"); + response.setContentType("text/html;charset=UTF-8"); + response.getWriter().write(headers.toString()); } else if (Objects.equals(request.getParameter("fileType"), "text")) { - CSVParser parser = new CSVParserBuilder().withSeparator(request.getParameter("dataSeparator").charAt(0)).build(); - + Set headersSet = new HashSet<>(); + String sep = request.getParameter("dataSeparator"); + char separator = (sep != null && !sep.isEmpty()) ? sep.charAt(0) : ','; + CSVParser parser = new CSVParserBuilder().withSeparator(separator).build(); try (CSVReader reader = new CSVReaderBuilder(new InputStreamReader(fileInputStream)).withCSVParser(parser).build()) { - String[] headers = reader.readNext(); // Read the first line which contains headers - + String[] headers = reader.readNext(); if (headers != null) { - // Print each header headersSet.addAll(Arrays.asList(headers)); } else { logger.info("File is empty or does not contain headers."); } } catch (IOException | CsvValidationException e) { - logger.error("An error occurred during extraction of headers.",e); + logger.error("An error occurred during extraction of headers.", e); } - + headersSet = headersSet.stream().sorted().collect(Collectors.toCollection(LinkedHashSet::new)); + StringBuilder headers = new StringBuilder(); + headers.append(" \n"); + response.setContentType("text/html;charset=UTF-8"); + response.getWriter().write(headers.toString()); } } - headersSet = headersSet.stream().sorted().collect(Collectors.toCollection(LinkedHashSet::new)); - StringBuilder headers = new StringBuilder(); - headers.append(" \n"); - response.setCharacterEncoding("UTF-8"); - response.setContentType("text/html;charset=UTF-8"); - - response.getWriter().write(headers.toString()); response.setHeader("updatedMap", ""); - - dataImporterForm.getColumnPairs().clear(); - } return null; } @@ -144,13 +192,20 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet String columnName = request.getParameter("columnName"); String selectedField = request.getParameter("selectedField"); - dataImporterForm.getColumnPairs().put(columnName, selectedField); - logger.info("Column Pairs:" + dataImporterForm.getColumnPairs()); + String configName = request.getParameter("configName"); + Map columnPairs = dataImporterForm.getColumnPairs(); + + if (configName != null && !configName.trim().isEmpty()) { + addColumnPairToConfig(configName.trim(), columnName, selectedField); + columnPairs = getConfigByName(configName.trim()); + } else { + columnPairs.put(columnName, selectedField); + } + logger.info("Column Pairs:" + columnPairs); ObjectMapper objectMapper = new ObjectMapper(); - String json = objectMapper.writeValueAsString(dataImporterForm.getColumnPairs()); + String json = objectMapper.writeValueAsString(columnPairs); - // Send response response.setContentType("application/json"); response.getWriter().write(json); response.setCharacterEncoding("UTF-8"); @@ -165,14 +220,20 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet String columnName = request.getParameter("columnName"); String selectedField = request.getParameter("selectedField"); - dataImporterForm.getColumnPairs().put(columnName, selectedField); - removeMapItem(dataImporterForm.getColumnPairs(), columnName, selectedField); - logger.info("Column Pairs:" + dataImporterForm.getColumnPairs()); + String configName = request.getParameter("configName"); + Map columnPairs = dataImporterForm.getColumnPairs(); + + if (configName != null && !configName.trim().isEmpty()) { + removeColumnPairFromConfig(configName.trim(), columnName); + columnPairs = getConfigByName(configName.trim()); + } else { + removeMapItem(columnPairs, columnName, selectedField); + } + logger.info("Column Pairs:" + columnPairs); ObjectMapper objectMapper = new ObjectMapper(); - String json = objectMapper.writeValueAsString(dataImporterForm.getColumnPairs()); + String json = objectMapper.writeValueAsString(columnPairs); - // Send response response.setContentType("application/json"); response.setCharacterEncoding("UTF-8"); response.getWriter().write(json); @@ -181,6 +242,34 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet } + if (Objects.equals(request.getParameter("action"), "getDataFileSheets")) { + logger.info("This is the action getDataFileSheets"); + if (dataImporterForm.getDataFile() == null || dataImporterForm.getDataFile().getFileSize() == 0) { + response.setStatus(400); + response.setContentType("application/json"); + response.getWriter().write("{\"error\":\"No file provided\"}"); + return null; + } + String fileType = request.getParameter("fileType"); + if (!Objects.equals(fileType, "excel")) { + response.setContentType("application/json"); + new ObjectMapper().writeValue(response.getWriter(), Collections.emptyList()); + return null; + } + List sheetNames = new ArrayList<>(); + try (InputStream is = dataImporterForm.getDataFile().getInputStream(); + Workbook workbook = new XSSFWorkbook(is)) { + int n = workbook.getNumberOfSheets(); + for (int i = 0; i < n; i++) { + sheetNames.add(workbook.getSheetAt(i).getSheetName()); + } + } + response.setContentType("application/json"); + response.setCharacterEncoding("UTF-8"); + new ObjectMapper().writeValue(response.getWriter(), sheetNames); + return null; + } + if (Objects.equals(request.getParameter("action"), "uploadDataFile")) { logger.info("This is the action " + request.getParameter("action")); Instant start = Instant.now(); @@ -213,8 +302,17 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet } } } - if (dataImporterForm.getColumnPairs().isEmpty() ||(!dataImporterForm.getColumnPairs().containsValue("Project Title") && !dataImporterForm.getColumnPairs().containsValue("Project Code"))) { - + // Resolve which config to use: saved config by name (from load/edit) or form's column pairs + String existingConfig = request.getParameter("existingConfig"); + String configNameParam = request.getParameter("configName"); + String configNameToUse = (configNameParam != null && !configNameParam.trim().isEmpty()) + ? configNameParam.trim() + : (existingConfig != null && !existingConfig.isEmpty() && !"0".equals(existingConfig) && !"1".equals(existingConfig) ? existingConfig.trim() : null); + Map columnPairsToUse = (configNameToUse != null) + ? getConfigByName(configNameToUse) + : dataImporterForm.getColumnPairs(); + + if (columnPairsToUse.isEmpty() || (!columnPairsToUse.containsValue(ImporterConstants.PROJECT_TITLE) && !columnPairsToUse.containsValue(ImporterConstants.PROJECT_CODE))) { response.setHeader("errorMessage", "You must have at least the 'Project Title' or 'Project Code' column in your config."); response.setStatus(400); return mapping.findForward("importData"); @@ -227,10 +325,8 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet return mapping.findForward("importData"); } else { - // Proceed with processing the file - String existingConfig = request.getParameter("existingConfig"); - logger.info("Existing configuration: {}",existingConfig); - if (!Objects.equals(existingConfig, "1")) { + logger.info("Existing configuration: {}", existingConfig); + if (configNameToUse == null) { saveImportConfig(request, fileName, dataImporterForm.getColumnPairs()); } @@ -239,17 +335,19 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet logger.info("Saved file record: {}",importedFilesRecord); boolean isInternal= dataImporterForm.isInternal(); logger.info("Internal: "+ isInternal); - if (isInternal) - { - dataImporterForm.getColumnPairs().put("Donor Agency", "Donor Agency"); + if (isInternal) { + columnPairsToUse = new HashMap<>(columnPairsToUse); + columnPairsToUse.put("Donor Agency", "Donor Agency"); } - logger.info("Configuration"+ dataImporterForm.getColumnPairs()); + logger.info("Configuration: {}", columnPairsToUse); if ((Objects.equals(request.getParameter("fileType"), "excel") || Objects.equals(request.getParameter("fileType"), "csv"))) { - + String dataSheetChoice = request.getParameter("dataSheetChoice"); + String dataSheetName = request.getParameter("dataSheetName"); + boolean useSpecificSheet = "sheet".equals(dataSheetChoice) && dataSheetName != null && !dataSheetName.trim().isEmpty(); // Process the file in batches - res = processExcelFileInBatches(importedFilesRecord, tempFile, request, dataImporterForm.getColumnPairs(), isInternal); + res = processExcelFileInBatches(importedFilesRecord, tempFile, request, columnPairsToUse, isInternal, useSpecificSheet ? dataSheetName : null); } else if ( Objects.equals(request.getParameter("fileType"), "text")) { - res=TxtDataImporter.processTxtFileInBatches(importedFilesRecord, tempFile, request, dataImporterForm.getColumnPairs(), isInternal); + res = TxtDataImporter.processTxtFileInBatches(importedFilesRecord, tempFile, request, columnPairsToUse, isInternal); } if (res != 1) { // Handle error @@ -322,6 +420,50 @@ private static Map getConfigByName(String configName) { return configValues; } + private static DataImporterConfig getConfigEntityByName(String configName) { + Session session = PersistenceManager.getRequestDBSession(); + String hql = "FROM DataImporterConfig WHERE configName = :configName"; + Query query = session.createQuery(hql); + query.setParameter("configName", configName, StringType.INSTANCE); + query.setMaxResults(1); + List list = query.list(); + return list.isEmpty() ? null : list.get(0); + } + + private static void addColumnPairToConfig(String configName, String columnName, String selectedField) { + DataImporterConfig config = getConfigEntityByName(configName); + if (config == null) { + logger.warn("Config not found for name: {}", configName); + return; + } + Session session = PersistenceManager.getRequestDBSession(); + DataImporterConfigValues cv = new DataImporterConfigValues(columnName, selectedField, config); + session.save(cv); + config.getConfigValues().add(cv); + session.flush(); + } + + private static void removeColumnPairFromConfig(String configName, String columnName) { + DataImporterConfig config = getConfigEntityByName(configName); + if (config == null) { + logger.warn("Config not found for name: {}", configName); + return; + } + DataImporterConfigValues toRemove = null; + for (DataImporterConfigValues v : config.getConfigValues()) { + if (columnName.equals(v.getConfigKey())) { + toRemove = v; + break; + } + } + if (toRemove != null) { + config.getConfigValues().remove(toRemove); + Session session = PersistenceManager.getRequestDBSession(); + session.delete(toRemove); + session.flush(); + } + } + public static void saveImportConfig(HttpServletRequest request, String fileName, Map config) { logger.info("Saving import config"); @@ -386,37 +528,55 @@ public static void saveImportConfig(HttpServletRequest request, String fileName, private List getEntityFieldsInfo() { List fieldsInfos = new ArrayList<>(); - fieldsInfos.add("Project Title"); - fieldsInfos.add("Project Code"); - fieldsInfos.add("Project Description"); - fieldsInfos.add("Primary Sector"); - fieldsInfos.add("Secondary Sector"); - fieldsInfos.add("Project Location"); - fieldsInfos.add("Project Start Date"); - fieldsInfos.add("Project End Date"); - fieldsInfos.add("Donor Agency"); - fieldsInfos.add("Exchange Rate"); - fieldsInfos.add("Donor Agency Code"); - fieldsInfos.add("Responsible Organization"); - fieldsInfos.add("Responsible Organization Code"); - fieldsInfos.add("Executing Agency"); - fieldsInfos.add("Implementing Agency"); - fieldsInfos.add("Actual Disbursement"); - fieldsInfos.add("Actual Commitment"); - fieldsInfos.add("Actual Expenditure"); - fieldsInfos.add("Planned Disbursement"); - fieldsInfos.add("Planned Commitment"); - fieldsInfos.add("Planned Expenditure"); - fieldsInfos.add("Funding Item"); - fieldsInfos.add("Transaction Date"); - fieldsInfos.add("Financing Instrument"); - fieldsInfos.add("Type Of Assistance"); - fieldsInfos.add("Secondary Subsector"); - fieldsInfos.add("Primary Subsector"); - fieldsInfos.add("Currency"); - fieldsInfos.add("Component Name"); - fieldsInfos.add("Component Code"); - fieldsInfos.add("Beneficiary Agency"); + fieldsInfos.add(ImporterConstants.PROJECT_TITLE); + fieldsInfos.add(ImporterConstants.PROJECT_CODE); + fieldsInfos.add(ImporterConstants.OBJECTIVE); + fieldsInfos.add(ImporterConstants.PROJECT_DESCRIPTION); + fieldsInfos.add(ImporterConstants.PRIMARY_SECTOR); + fieldsInfos.add(ImporterConstants.SECONDARY_SECTOR); + fieldsInfos.add(ImporterConstants.PROJECT_LOCATION); + fieldsInfos.add(ImporterConstants.PROJECT_START_DATE); + fieldsInfos.add(ImporterConstants.PROJECT_END_DATE); + fieldsInfos.add(ImporterConstants.DONOR_AGENCY); + fieldsInfos.add(ImporterConstants.EXCHANGE_RATE); + fieldsInfos.add(ImporterConstants.DONOR_AGENCY_CODE); + fieldsInfos.add(ImporterConstants.RESPONSIBLE_ORGANIZATION); + fieldsInfos.add(ImporterConstants.RESPONSIBLE_ORGANIZATION_CODE); + fieldsInfos.add(ImporterConstants.EXECUTING_AGENCY); + fieldsInfos.add(ImporterConstants.IMPLEMENTING_AGENCY); + fieldsInfos.add(ImporterConstants.ACTUAL_DISBURSEMENT); + fieldsInfos.add(ImporterConstants.ACTUAL_COMMITMENT); + fieldsInfos.add(ImporterConstants.ACTUAL_EXPENDITURE); + fieldsInfos.add(ImporterConstants.PLANNED_DISBURSEMENT); + fieldsInfos.add(ImporterConstants.PLANNED_COMMITMENT); + fieldsInfos.add(ImporterConstants.PLANNED_EXPENDITURE); + fieldsInfos.add(ImporterConstants.TRANSACTION_AMOUNT); + fieldsInfos.add(ImporterConstants.MEASURE_TYPE); + fieldsInfos.add(ImporterConstants.TRANSACTION_DATE); + fieldsInfos.add(ImporterConstants.FINANCING_INSTRUMENT); + fieldsInfos.add(ImporterConstants.TYPE_OF_ASSISTANCE); + fieldsInfos.add(ImporterConstants.SECONDARY_SUBSECTOR); + fieldsInfos.add(ImporterConstants.PRIMARY_SUBSECTOR); + fieldsInfos.add(ImporterConstants.CURRENCY); + fieldsInfos.add(ImporterConstants.COMPONENT_NAME); + fieldsInfos.add(ImporterConstants.COMPONENT_CODE); + fieldsInfos.add(ImporterConstants.BENEFICIARY_AGENCY); + fieldsInfos.add(ImporterConstants.PROJECT_STATUS); + // Indicator columns for M&E import + fieldsInfos.add(ImporterConstants.INDICATOR_NAME); + fieldsInfos.add(ImporterConstants.PROGRAM_NAME); + fieldsInfos.add(ImporterConstants.INDICATOR_LOCATION); + fieldsInfos.add(ImporterConstants.ORIGINAL_BASE_VALUE); + fieldsInfos.add(ImporterConstants.ORIGINAL_BASE_VALUE_DATE); + fieldsInfos.add(ImporterConstants.REVISED_BASE_VALUE); + fieldsInfos.add(ImporterConstants.REVISED_BASE_VALUE_DATE); + fieldsInfos.add(ImporterConstants.ORIGINAL_TARGET_VALUE); + fieldsInfos.add(ImporterConstants.ORIGINAL_TARGET_VALUE_DATE); + fieldsInfos.add(ImporterConstants.REVISED_TARGET_VALUE); + fieldsInfos.add(ImporterConstants.REVISED_TARGET_VALUE_DATE); + fieldsInfos.add(ImporterConstants.ACTUAL_VALUE); + fieldsInfos.add(ImporterConstants.ACTUAL_VALUE_DATE); + fieldsInfos.add(ImporterConstants.UNIT_OF_MEASURE); return fieldsInfos.stream().sorted().collect(Collectors.toList()); } diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java index d41c5e64c98..7ab51d85db4 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java @@ -1,6 +1,17 @@ package org.digijava.module.aim.action.dataimporter; -import com.fasterxml.jackson.core.JsonProcessingException; +import java.io.File; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import javax.servlet.http.HttpServletRequest; + import org.apache.poi.openxml4j.exceptions.InvalidFormatException; import org.apache.poi.openxml4j.exceptions.InvalidOperationException; import org.apache.poi.ss.usermodel.Cell; @@ -15,7 +26,21 @@ import org.digijava.module.aim.action.dataimporter.dbentity.ImportedProject; import org.digijava.module.aim.action.dataimporter.model.Funding; import org.digijava.module.aim.action.dataimporter.model.ImportDataModel; +import org.digijava.module.aim.action.dataimporter.util.ImporterConstants; import org.digijava.module.aim.action.dataimporter.util.ImportedFileUtil; +import org.digijava.module.aim.action.dataimporter.util.ImporterUtil; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.existingActivity; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.getColumnIndexByName; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.getKey; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.getStringValueFromCell; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.parseMeasureType; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.addIndicatorDataToActivity; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.importTheData; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.setAFundingItemForExcel; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.setStatus; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.updateLocations; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.updateOrgs; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.updateSectors; import org.digijava.module.aim.dbentity.AmpActivityVersion; import org.digijava.module.aim.util.FeaturesUtil; import org.digijava.module.aim.util.TeamMemberUtil; @@ -23,44 +48,48 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.http.HttpServletRequest; -import java.io.File; -import java.io.IOException; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.*; +import com.fasterxml.jackson.core.JsonProcessingException; public class ExcelImporter { static Logger logger = LoggerFactory.getLogger(ExcelImporter.class); private static final int BATCH_SIZE = 1000; public static int processExcelFileInBatches(ImportedFilesRecord importedFilesRecord, File file, HttpServletRequest request, Map config, boolean isInternal) { - int res=0; + return processExcelFileInBatches(importedFilesRecord, file, request, config, isInternal, null); + } + + public static int processExcelFileInBatches(ImportedFilesRecord importedFilesRecord, File file, HttpServletRequest request, Map config, boolean isInternal, String sheetNameToProcess) { + int res = 0; ImportedFileUtil.updateFileStatus(importedFilesRecord, ImportStatus.IN_PROGRESS); try (Workbook workbook = new XSSFWorkbook(file)) { int numberOfSheets = workbook.getNumberOfSheets(); logger.info("Number of sheets: {}", numberOfSheets); - // Process each sheet in the workbook - for (int i = 0; i < numberOfSheets; i++) { - logger.info("Sheet number: {}", i); - Sheet sheet = workbook.getSheetAt(i); + if (sheetNameToProcess != null && !sheetNameToProcess.trim().isEmpty()) { + Sheet sheet = workbook.getSheet(sheetNameToProcess); + if (sheet == null) { + logger.error("Sheet not found: {}", sheetNameToProcess); + ImportedFileUtil.updateFileStatus(importedFilesRecord, ImportStatus.FAILED); + return 0; + } if (isInternal) { addDonorAgencyColumn(sheet, FeaturesUtil.getGlobalSettingValue("Internal Ecowas Donor")); - } - - processSheetInBatches(sheet, request,config, importedFilesRecord); + processSheetInBatches(sheet, request, config, importedFilesRecord); + } else { + // Process each sheet in the workbook + for (int i = 0; i < numberOfSheets; i++) { + logger.info("Sheet number: {}", i); + Sheet sheet = workbook.getSheetAt(i); + if (isInternal) { + addDonorAgencyColumn(sheet, FeaturesUtil.getGlobalSettingValue("Internal Ecowas Donor")); + } + processSheetInBatches(sheet, request, config, importedFilesRecord); + } } logger.info("Closing the workbook..."); - res =1; + res = 1; } catch (IOException e) { ImportedFileUtil.updateFileStatus(importedFilesRecord, ImportStatus.FAILED); logger.error("Error processing Excel file: {}", e.getMessage(), e); @@ -129,129 +158,183 @@ public static void processSheetInBatches(Sheet sheet, HttpServletRequest request public static void processBatch(List batch,Sheet sheet, HttpServletRequest request, Map config, ImportedFilesRecord importedFilesRecord) throws JsonProcessingException { // Process the batch of rows SessionUtil.extendSessionIfNeeded(request); - Session session = PersistenceManager.getRequestDBSession(); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); for (Row row : batch) { if (row != null) { + final Row rowRef = row; ImportedProject importedProject = new ImportedProject(); importedProject.setImportedFilesRecord(importedFilesRecord); List fundings = new ArrayList<>(); ImportDataModel importDataModel = new ImportDataModel(); importDataModel.setModified_by(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeamMemId()); - importDataModel.setCreated_by(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeamMemId()); + // created_by is set in ensureCreatedBySet when building the API map (correct for new vs existing) importDataModel.setTeam(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeam().getAmpTeamId()); importDataModel.setIs_draft(true); OffsetDateTime now = OffsetDateTime.now(ZoneOffset.UTC); importDataModel.setCreation_date(now.format(formatter)); - setStatus(importDataModel); - int componentCodeColumn = getColumnIndexByName(sheet, getKey(config, "Component Code")); - String componentCode = componentCodeColumn >= 0 ? getStringValueFromCell(row.getCell(componentCodeColumn),true) : null; + int componentCodeColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.COMPONENT_CODE)); + String componentCode = componentCodeColumn >= 0 ? getStringValueFromCell(rowRef.getCell(componentCodeColumn),true) : null; - int componentNameColumn = getColumnIndexByName(sheet, getKey(config, "Component Name")); - String componentName = componentNameColumn >= 0 ? getStringValueFromCell(row.getCell(componentNameColumn),true): null; + int componentNameColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.COMPONENT_NAME)); + String componentName = componentNameColumn >= 0 ? getStringValueFromCell(rowRef.getCell(componentNameColumn),true): null; - int donorAgencyCodeColumn = getColumnIndexByName(sheet, getKey(config, "Donor Agency Code")); - String donorAgencyCode = donorAgencyCodeColumn >= 0 ? getStringValueFromCell(row.getCell(donorAgencyCodeColumn),true) : null; + int donorAgencyCodeColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.DONOR_AGENCY_CODE)); + String donorAgencyCode = donorAgencyCodeColumn >= 0 ? getStringValueFromCell(rowRef.getCell(donorAgencyCodeColumn),true) : null; - int responsibleOrgCodeColumn = getColumnIndexByName(sheet, getKey(config, "Responsible Organization Code")); - String responsibleOrgCode = responsibleOrgCodeColumn >= 0 ? getStringValueFromCell(row.getCell(responsibleOrgCodeColumn),true) : null; + int responsibleOrgCodeColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.RESPONSIBLE_ORGANIZATION_CODE)); + String responsibleOrgCode = responsibleOrgCodeColumn >= 0 ? getStringValueFromCell(rowRef.getCell(responsibleOrgCodeColumn),true) : null; - int primarySubSectorColumn = getColumnIndexByName(sheet, getKey(config, "Primary Subsector")); - String primarySubSector = primarySubSectorColumn >= 0 ? getStringValueFromCell(row.getCell(primarySubSectorColumn),true) : null; + int primarySubSectorColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.PRIMARY_SUBSECTOR)); + String primarySubSector = primarySubSectorColumn >= 0 ? getStringValueFromCell(rowRef.getCell(primarySubSectorColumn),true) : null; - int secondarySubSectorColumn = getColumnIndexByName(sheet, getKey(config, "Secondary Subsector")); - String secondarySubSector = secondarySubSectorColumn >= 0 ? getStringValueFromCell(row.getCell(secondarySubSectorColumn),true) : null; + int secondarySubSectorColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.SECONDARY_SUBSECTOR)); + String secondarySubSector = secondarySubSectorColumn >= 0 ? getStringValueFromCell(rowRef.getCell(secondarySubSectorColumn),true) : null; - int projectCodeColumn = getColumnIndexByName(sheet, getKey(config, "Project Code")); - String projectCode = projectCodeColumn >= 0 ? getStringValueFromCell(row.getCell(projectCodeColumn),false) : ""; + int projectCodeColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.PROJECT_CODE)); + String projectCode = projectCodeColumn >= 0 ? getStringValueFromCell(rowRef.getCell(projectCodeColumn),false) : ""; importDataModel.setProject_code(projectCode); - int projectTitleColumn = getColumnIndexByName(sheet, getKey(config, "Project Title")); - String projectTitle = projectTitleColumn >= 0 ? getStringValueFromCell(row.getCell(projectTitleColumn),false) : ""; + int projectTitleColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.PROJECT_TITLE)); + String projectTitle = projectTitleColumn >= 0 ? getStringValueFromCell(rowRef.getCell(projectTitleColumn),false) : ""; importDataModel.setProject_title(projectTitle); + int objectiveColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.OBJECTIVE)); + String objective = objectiveColumn >= 0 ? getStringValueFromCell(rowRef.getCell(objectiveColumn),false) : null; + importDataModel.setObjective(objective); - int projectDescColumn = getColumnIndexByName(sheet, getKey(config, "Project Description")); - String projectDesc = projectDescColumn >= 0 ? getStringValueFromCell(row.getCell(projectDescColumn),false) : null; + int projectDescColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.PROJECT_DESCRIPTION)); + String projectDesc = projectDescColumn >= 0 ? getStringValueFromCell(rowRef.getCell(projectDescColumn),false) : null; importDataModel.setDescription(projectDesc); - AmpActivityVersion existing = existingActivity(projectTitle, projectCode, session); - Long responsibleOrgId = null; - -// if (existing!=null && SKIP_EXISTING) -// { -// logger.info("Instructed to skip existing activities"); -// importedProject.setImportStatus(ImportStatus.SKIPPED); -// continue; -// } - - logger.info("Row Number: {}, Sheet Name: {}", row.getRowNum(), sheet.getSheetName()); - for (Map.Entry entry : config.entrySet()) { - Funding fundingItem = new Funding(); - - int columnIndex = getColumnIndexByName(sheet, entry.getKey()); - - if (columnIndex >= 0) { - Cell cell = row.getCell(columnIndex); - switch (entry.getValue()) { - case "Project Location": - updateLocations(importDataModel,Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(),session); - break; - case "Primary Sector": - updateSectors(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), session, true, primarySubSector); - break; - case "Secondary Sector": - updateSectors(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), session, false, secondarySubSector); - break; - case "Donor Agency": - logger.info("Getting donor"); - updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), donorAgencyCode, session, "donor"); - break; - case "Responsible Organization": - responsibleOrgId = updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), responsibleOrgCode, session, "responsibleOrg"); - break; - case "Beneficiary Agency": - responsibleOrgId = updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), responsibleOrgCode, session, "beneficiaryAgency"); - break; - case "Funding Item": - setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, true, true, false,"Actual", fundingItem, existing); - break; - case "Planned Commitment": - setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, true, false,false, "Planned", fundingItem, existing); - break; - case "Planned Disbursement": - setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, false, true, false,"Planned", fundingItem, existing); - break; - case "Planned Expenditure": - setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, false, false,true, "Planned", fundingItem, existing); - break; - case "Actual Commitment": - setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, true, false, false,"Actual", fundingItem, existing); - break; - case "Actual Disbursement": - setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, false, true, false,"Actual", fundingItem, existing); - break; - case "Actual Expenditure": - setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, false, false,true, "Actual", fundingItem, existing); - break; - case "Reporting Date": - default: - logger.error("Unexpected value: " + entry.getValue()); - break; - + // Use holder arrays to capture values from lambda (for effectively final requirement) + final Long[] existingActivityIdHolder = new Long[1]; // Store only the ID, not the entity + final Long[] responsibleOrgIdHolder = new Long[1]; + + // Phase 1: Data preparation - use independent transaction/session + try { + PersistenceManager.doInTransaction(session -> { + + if (config.containsValue(ImporterConstants.PROJECT_STATUS)) { + String projectStatusStr = ImporterUtil.getCellValueByConfig(rowRef, sheet, config, ImporterConstants.PROJECT_STATUS); + if (projectStatusStr != null && !projectStatusStr.trim().isEmpty()) { + Long statusId = ImporterUtil.getOrCreateActivityStatusCategoryValue(projectStatusStr.trim(), session); + if (statusId != null) { + importDataModel.setActivity_status(statusId); + } + } } - - + setStatus(importDataModel); + + AmpActivityVersion existing = existingActivity(projectTitle, projectCode, session); + existingActivityIdHolder[0] = existing != null ? existing.getAmpActivityId() : null; + + logger.info("Row Number: {}, Sheet Name: {}", rowRef.getRowNum(), sheet.getSheetName()); + for (Map.Entry entry : config.entrySet()) { + Funding fundingItem = new Funding(); + + int columnIndex = getColumnIndexByName(sheet, entry.getKey()); + + if (columnIndex >= 0) { + Cell cell = rowRef.getCell(columnIndex); + switch (entry.getValue()) { + case ImporterConstants.PROJECT_LOCATION: + updateLocations(importDataModel,Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(),session); + break; + case ImporterConstants.PRIMARY_SECTOR: + updateSectors(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), session, true, primarySubSector); + break; + case ImporterConstants.SECONDARY_SECTOR: + updateSectors(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), session, false, secondarySubSector); + break; + case ImporterConstants.DONOR_AGENCY: + logger.info("Getting donor"); + updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), donorAgencyCode, session, ImporterConstants.ORG_TYPE_DONOR); + break; + case ImporterConstants.RESPONSIBLE_ORGANIZATION: + responsibleOrgIdHolder[0] = updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), responsibleOrgCode, session, ImporterConstants.ORG_TYPE_RESPONSIBLE_ORG); + break; + case ImporterConstants.BENEFICIARY_AGENCY: + responsibleOrgIdHolder[0] = updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), responsibleOrgCode, session, ImporterConstants.ORG_TYPE_BENEFICIARY_AGENCY); + break; + case ImporterConstants.TRANSACTION_AMOUNT: { + boolean commitment = true, disbursement = true, expenditure = false; + String adjustmentType = ImporterConstants.ADJUSTMENT_TYPE_ACTUAL; + if (config.containsValue(ImporterConstants.MEASURE_TYPE)) { + String measureTypeStr = ImporterUtil.getCellValueByConfig(rowRef, sheet, config, ImporterConstants.MEASURE_TYPE); + ImporterUtil.MeasureTypeResult parsed = parseMeasureType(measureTypeStr); + if (parsed != null) { + commitment = parsed.commitment; + disbursement = parsed.disbursement; + expenditure = parsed.expenditure; + adjustmentType = parsed.adjustmentType; + } + } + setAFundingItemForExcel(sheet, config, rowRef, entry, importDataModel, session, cell, commitment, disbursement, expenditure, adjustmentType, fundingItem, null); + break; + } + case ImporterConstants.PLANNED_COMMITMENT: + setAFundingItemForExcel(sheet, config, rowRef, entry, importDataModel, session, cell, true, false,false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED, fundingItem, null); + break; + case ImporterConstants.PLANNED_DISBURSEMENT: + setAFundingItemForExcel(sheet, config, rowRef, entry, importDataModel, session, cell, false, true, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED, fundingItem, null); + break; + case ImporterConstants.PLANNED_EXPENDITURE: + setAFundingItemForExcel(sheet, config, rowRef, entry, importDataModel, session, cell, false, false,true, ImporterConstants.ADJUSTMENT_TYPE_PLANNED, fundingItem, null); + break; + case ImporterConstants.ACTUAL_COMMITMENT: + setAFundingItemForExcel(sheet, config, rowRef, entry, importDataModel, session, cell, true, false, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL, fundingItem, null); + break; + case ImporterConstants.ACTUAL_DISBURSEMENT: + setAFundingItemForExcel(sheet, config, rowRef, entry, importDataModel, session, cell, false, true, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL, fundingItem, null); + break; + case ImporterConstants.ACTUAL_EXPENDITURE: + setAFundingItemForExcel(sheet, config, rowRef, entry, importDataModel, session, cell, false, false,true, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL, fundingItem, null); + break; + case ImporterConstants.MEASURE_TYPE: + break; + case ImporterConstants.PROJECT_STATUS: + break; + case ImporterConstants.REPORTING_DATE: + default: + logger.error("Unexpected value: " + entry.getValue()); + break; + } + } + fundings.add(fundingItem); + } + }); + } catch (RuntimeException e) { + Throwable cause = e.getCause(); + if (cause instanceof JsonProcessingException) { + throw (JsonProcessingException) cause; } - fundings.add(fundingItem); - - + logger.error("Error preparing data for row " + rowRef.getRowNum() + " in sheet " + sheet.getSheetName() + ": " + e.getMessage(), e); } + // Phase 2: Activity import - DO NOT wrap in transaction, let ActivityGatekeeper handle it + // This avoids nested transaction issues when ActivityGatekeeper.doWithLock creates its own transaction + Long activityId; + try { + // Pass only the ID, not the entity - importTheData will re-fetch in its own transaction context + activityId = importTheData(importDataModel, null, importedProject, componentName, componentCode, responsibleOrgIdHolder[0], fundings, existingActivityIdHolder[0]); + } catch (JsonProcessingException e) { + throw e; + } - importTheData(importDataModel, session, importedProject, componentName, componentCode, responsibleOrgId, fundings, existing); - + // Phase 3: Indicator import - use independent transaction/session + if (activityId != null && config.containsValue(ImporterConstants.INDICATOR_NAME)) { + logger.info("Adding indicator data for activity " + activityId); + try { + final Long activityIdFinal = activityId; + PersistenceManager.doInTransaction(s -> { + addIndicatorDataToActivity(activityIdFinal, rowRef, sheet, config, s); + logger.info("Indicator data added for activity " + activityIdFinal); + }); + } catch (Exception e) { + logger.error("Failed to add indicator data for activity " + activityId, e); + } + } } } } diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java index 1c735c6f9bb..2ecb1fdbed9 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java @@ -12,7 +12,9 @@ import org.digijava.module.aim.action.dataimporter.dbentity.ImportedFilesRecord; import org.digijava.module.aim.action.dataimporter.dbentity.ImportedProject; import org.digijava.module.aim.action.dataimporter.model.Funding; +import org.digijava.module.aim.action.dataimporter.util.ImporterConstants; import org.digijava.module.aim.action.dataimporter.model.ImportDataModel; +import org.digijava.module.aim.action.dataimporter.util.ImporterUtil; import org.digijava.module.aim.dbentity.AmpActivityVersion; import org.digijava.module.aim.util.FeaturesUtil; import org.digijava.module.aim.util.TeamMemberUtil; @@ -80,100 +82,146 @@ public static int processTxtFileInBatches(ImportedFilesRecord importedFilesRecor private static void processBatch(List> batch, HttpServletRequest request,Map config, ImportedFilesRecord importedFilesRecord) throws JsonProcessingException { logger.info("Processing txt batch"); SessionUtil.extendSessionIfNeeded(request); - Session session = PersistenceManager.getRequestDBSession(); DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"); for (Map row : batch) { + final Map rowRef = row; ImportedProject importedProject= new ImportedProject(); importedProject.setImportedFilesRecord(importedFilesRecord); List fundings= new ArrayList<>(); - ImportDataModel importDataModel = new ImportDataModel(); importDataModel.setModified_by(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeamMemId()); - importDataModel.setCreated_by(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeamMemId()); importDataModel.setTeam(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeam().getAmpTeamId()); importDataModel.setIs_draft(true); OffsetDateTime now = OffsetDateTime.now(ZoneOffset.UTC); importDataModel.setCreation_date(now.format(formatter)); - setStatus(importDataModel); - String componentName= row.get(getKey(config, "Component Name")); - String componentCode= row.get(getKey(config, "Component Code")); - String projectCode= row.get(getKey(config, "Project Code")); - String projectTitle= row.get(getKey(config, "Project Title")); - String projectDesc= row.get(getKey(config, "Project Description")); - String primarySubSector= row.get(getKey(config, "Primary Subsector")); - String secondarySubSector= row.get(getKey(config, "Secondary Subsector")); - AmpActivityVersion existing = existingActivity(projectTitle,projectCode,session); - if (existing!=null && SKIP_EXISTING) - { - logger.info("Instructed to skip existing activities"); - importedProject.setImportStatus(ImportStatus.SKIPPED); - continue; - } - - importDataModel.setProject_title(projectTitle); - importDataModel.setProject_code(projectCode); - importDataModel.setDescription(projectDesc); - - String donorAgencyCode= row.get(getKey(config, "Donor Agency Code")); - String responsibleOrgCode= row.get(getKey(config, "Responsible Organization Code")); - Long responsibleOrgId=null; - - logger.info("Configuration: "+config); - for (Map.Entry entry : config.entrySet()) { - Funding fundingItem = new Funding(); - switch (entry.getValue()) { - case "Project Location": - updateLocations(importDataModel, row.get(entry.getKey().trim()),session); - break; - case "Primary Sector": - updateSectors(importDataModel, row.get(entry.getKey().trim()), session, true, primarySubSector); - break; - case "Secondary Sector": - updateSectors(importDataModel, row.get(entry.getKey().trim()), session, false, secondarySubSector); - break; - case "Donor Agency": - updateOrgs(importDataModel,row.get(entry.getKey().trim()),donorAgencyCode, session, "donor"); - break; - case "Responsible Organization": - responsibleOrgId=updateOrgs(importDataModel,row.get(entry.getKey().trim()),responsibleOrgCode, session, "responsibleOrg"); - break; - case "Beneficiary Agency": - responsibleOrgId=updateOrgs(importDataModel,row.get(entry.getKey().trim()),responsibleOrgCode, session, "beneficiaryAgency"); - break; - case "Funding Item": - setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), true, true, false,"Actual", fundingItem, existing); - break; - case "Planned Commitment": - setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), true, false, false,"Planned", fundingItem, existing); - break; - case "Planned Disbursement": - setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), false, true, false,"Planned", fundingItem, existing); - break; - case "Planned Expenditure": - setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), false, false,true, "Planned", fundingItem, existing); - break; - case "Actual Commitment": - setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), true, false, false,"Actual", fundingItem, existing); - break; - case "Actual Disbursement": - setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), false, true, false,"Actual", fundingItem, existing); - break; - case "Actual Expenditure": - setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), false, false,true, "Actual", fundingItem, existing); - break; - default: - logger.error("Unexpected value: " + entry.getValue()); - break; + String componentName= rowRef.get(getKey(config, ImporterConstants.COMPONENT_NAME)); + String componentCode= rowRef.get(getKey(config, ImporterConstants.COMPONENT_CODE)); + String projectCode= rowRef.get(getKey(config, ImporterConstants.PROJECT_CODE)); + String projectTitle= rowRef.get(getKey(config, ImporterConstants.PROJECT_TITLE)); + String projectDesc= rowRef.get(getKey(config, ImporterConstants.PROJECT_DESCRIPTION)); + String objective= rowRef.get(getKey(config, ImporterConstants.OBJECTIVE)); + String primarySubSector= rowRef.get(getKey(config, ImporterConstants.PRIMARY_SUBSECTOR)); + String secondarySubSector= rowRef.get(getKey(config, ImporterConstants.SECONDARY_SUBSECTOR)); + String projectStatusStr = rowRef.get(getKey(config, ImporterConstants.PROJECT_STATUS)); + + // Use holder arrays to capture values from lambda (for effectively final requirement) + final Long[] existingActivityIdHolder = new Long[1]; // Store only the ID, not the entity + final Long[] responsibleOrgIdHolder = new Long[1]; + + // Phase 1: Data preparation - use independent transaction/session + try { + PersistenceManager.doInTransaction(session -> { + + AmpActivityVersion existing = existingActivity(projectTitle, projectCode, session); + existingActivityIdHolder[0] = existing != null ? existing.getAmpActivityId() : null; + if (existing != null && SKIP_EXISTING) { + logger.info("Instructed to skip existing activities"); + importedProject.setImportStatus(ImportStatus.SKIPPED); + return; + } + + importDataModel.setProject_title(projectTitle); + importDataModel.setObjective(objective); + importDataModel.setProject_code(projectCode); + importDataModel.setDescription(projectDesc); + + if (projectStatusStr != null && !projectStatusStr.trim().isEmpty()) { + Long statusId = getOrCreateActivityStatusCategoryValue(projectStatusStr.trim(), session); + if (statusId != null) { + importDataModel.setActivity_status(statusId); + } + } + setStatus(importDataModel); + + String donorAgencyCode = rowRef.get(getKey(config, ImporterConstants.DONOR_AGENCY_CODE)); + String responsibleOrgCode = rowRef.get(getKey(config, ImporterConstants.RESPONSIBLE_ORGANIZATION_CODE)); + + logger.info("Configuration: " + config); + for (Map.Entry entry : config.entrySet()) { + Funding fundingItem = new Funding(); + switch (entry.getValue()) { + case ImporterConstants.PROJECT_LOCATION: + updateLocations(importDataModel, rowRef.get(entry.getKey().trim()), session); + break; + case ImporterConstants.PRIMARY_SECTOR: + updateSectors(importDataModel, rowRef.get(entry.getKey().trim()), session, true, primarySubSector); + break; + case ImporterConstants.SECONDARY_SECTOR: + updateSectors(importDataModel, rowRef.get(entry.getKey().trim()), session, false, secondarySubSector); + break; + case ImporterConstants.DONOR_AGENCY: + updateOrgs(importDataModel, rowRef.get(entry.getKey().trim()), donorAgencyCode, session, ImporterConstants.ORG_TYPE_DONOR); + break; + case ImporterConstants.RESPONSIBLE_ORGANIZATION: + responsibleOrgIdHolder[0] = updateOrgs(importDataModel, rowRef.get(entry.getKey().trim()), responsibleOrgCode, session, ImporterConstants.ORG_TYPE_RESPONSIBLE_ORG); + break; + case ImporterConstants.BENEFICIARY_AGENCY: + responsibleOrgIdHolder[0] = updateOrgs(importDataModel, rowRef.get(entry.getKey().trim()), responsibleOrgCode, session, ImporterConstants.ORG_TYPE_BENEFICIARY_AGENCY); + break; + case ImporterConstants.TRANSACTION_AMOUNT: { + boolean commitment = true, disbursement = true, expenditure = false; + String adjustmentType = ImporterConstants.ADJUSTMENT_TYPE_ACTUAL; + if (config.containsValue(ImporterConstants.MEASURE_TYPE)) { + String measureTypeStr = rowRef.get(getKey(config, ImporterConstants.MEASURE_TYPE)); + ImporterUtil.MeasureTypeResult parsed = parseMeasureType(measureTypeStr); + if (parsed != null) { + commitment = parsed.commitment; + disbursement = parsed.disbursement; + expenditure = parsed.expenditure; + adjustmentType = parsed.adjustmentType; + } + } + setAFundingItemForTxt(config, rowRef, entry, importDataModel, session, Double.parseDouble(rowRef.get(entry.getKey().trim())), commitment, disbursement, expenditure, adjustmentType, fundingItem, null); + break; + } + case ImporterConstants.PLANNED_COMMITMENT: + setAFundingItemForTxt(config, rowRef, entry, importDataModel, session, Double.parseDouble(rowRef.get(entry.getKey().trim())), true, false, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED, fundingItem, null); + break; + case ImporterConstants.PLANNED_DISBURSEMENT: + setAFundingItemForTxt(config, rowRef, entry, importDataModel, session, Double.parseDouble(rowRef.get(entry.getKey().trim())), false, true, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED, fundingItem, null); + break; + case ImporterConstants.PLANNED_EXPENDITURE: + setAFundingItemForTxt(config, rowRef, entry, importDataModel, session, Double.parseDouble(rowRef.get(entry.getKey().trim())), false, false, true, ImporterConstants.ADJUSTMENT_TYPE_PLANNED, fundingItem, null); + break; + case ImporterConstants.ACTUAL_COMMITMENT: + setAFundingItemForTxt(config, rowRef, entry, importDataModel, session, Double.parseDouble(rowRef.get(entry.getKey().trim())), true, false, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL, fundingItem, null); + break; + case ImporterConstants.ACTUAL_DISBURSEMENT: + setAFundingItemForTxt(config, rowRef, entry, importDataModel, session, Double.parseDouble(rowRef.get(entry.getKey().trim())), false, true, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL, fundingItem, null); + break; + case ImporterConstants.ACTUAL_EXPENDITURE: + setAFundingItemForTxt(config, rowRef, entry, importDataModel, session, Double.parseDouble(rowRef.get(entry.getKey().trim())), false, false, true, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL, fundingItem, null); + break; + case ImporterConstants.MEASURE_TYPE: + break; + case ImporterConstants.PROJECT_STATUS: + break; + default: + logger.error("Unexpected value: " + entry.getValue()); + break; + } + fundings.add(fundingItem); + logger.info("Funding items :{}", fundings); + } + }); + } catch (RuntimeException e) { + Throwable cause = e.getCause(); + if (cause instanceof JsonProcessingException) { + throw (JsonProcessingException) cause; } - fundings.add(fundingItem); - logger.info("Funding items :{}",fundings); - + throw e; } - importTheData(importDataModel, session, importedProject, componentName, componentCode,responsibleOrgId,fundings,existing); - + // Phase 2: Activity import - DO NOT wrap in transaction, let ActivityGatekeeper handle it + // This avoids nested transaction issues when ActivityGatekeeper.doWithLock creates its own transaction + try { + // Pass only the ID, not the entity - importTheData will re-fetch in its own transaction context + importTheData(importDataModel, null, importedProject, componentName, componentCode, responsibleOrgIdHolder[0], fundings, existingActivityIdHolder[0]); + } catch (JsonProcessingException e) { + throw e; + } } } diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/dbentity/ImportedProjectCurrency.hbm.xml b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/dbentity/ImportedProjectCurrency.hbm.xml index 577485f4d84..4fc39b2a539 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/dbentity/ImportedProjectCurrency.hbm.xml +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/dbentity/ImportedProjectCurrency.hbm.xml @@ -10,7 +10,7 @@ IMPORTED_PROJECT_CURRENCY_ID - + diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/model/ImportDataModel.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/model/ImportDataModel.java index a88a1bc9ccd..82652e4368f 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/model/ImportDataModel.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/model/ImportDataModel.java @@ -51,8 +51,8 @@ public class ImportDataModel { private Object approval_date; private Integer approval_status; private Object archived; - private Set indicators; - private Set activity_documents; +// private Set indicators; +// private Set activity_documents; private Long activity_status; private Long activity_budget; private Long implementation_location; @@ -393,21 +393,21 @@ public void setArchived(Object archived) { this.archived = archived; } - public Set getIndicators() { - return indicators; - } +// public Set getIndicators() { +// return indicators; +// } - public void setIndicators(Set indicators) { - this.indicators = indicators; - } +// public void setIndicators(Set indicators) { +// this.indicators = indicators; +// } - public Set getActivity_documents() { - return activity_documents; - } +// public Set getActivity_documents() { +// return activity_documents; +// } - public void setActivity_documents(Set activity_documents) { - this.activity_documents = activity_documents; - } +// public void setActivity_documents(Set activity_documents) { +// this.activity_documents = activity_documents; +// } public Long getActivity_status() { return activity_status; @@ -545,8 +545,8 @@ public String toString() { ", approval_date=" + approval_date + ", approval_status=" + approval_status + ", archived=" + archived + - ", indicators=" + indicators + - ", activity_documents=" + activity_documents + +// ", indicators=" + indicators + +// ", activity_documents=" + activity_documents + ", activity_status=" + activity_status + ", activity_budget=" + activity_budget + ", implementation_level=" + implementation_level + diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterConstants.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterConstants.java new file mode 100644 index 00000000000..8b863b0b0d4 --- /dev/null +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterConstants.java @@ -0,0 +1,78 @@ +package org.digijava.module.aim.action.dataimporter.util; + +/** + * Central place for all string constants used by the Data Importer (Excel, Txt, config). + * These are the field names used in column-to-entity mapping and in switch/case logic. + */ +public final class ImporterConstants { + + private ImporterConstants() { + } + + // ----- Adjustment types (funding: Actual vs Planned) ----- + public static final String ADJUSTMENT_TYPE_ACTUAL = "Actual"; + public static final String ADJUSTMENT_TYPE_PLANNED = "Planned"; + + // ----- Organization role types (for updateOrgs) ----- + public static final String ORG_TYPE_DONOR = "donor"; + public static final String ORG_TYPE_RESPONSIBLE_ORG = "responsibleOrg"; + public static final String ORG_TYPE_BENEFICIARY_AGENCY = "beneficiaryAgency"; + + // ----- Entity / column field names (template mapping) ----- + public static final String PROJECT_TITLE = "Project Title"; + public static final String PROJECT_CODE = "Project Code"; + public static final String OBJECTIVE = "Objective"; + public static final String PROJECT_DESCRIPTION = "Project Description"; + public static final String PRIMARY_SECTOR = "Primary Sector"; + public static final String SECONDARY_SECTOR = "Secondary Sector"; + public static final String PROJECT_LOCATION = "Project Location"; + public static final String PROJECT_START_DATE = "Project Start Date"; + public static final String PROJECT_END_DATE = "Project End Date"; + public static final String DONOR_AGENCY = "Donor Agency"; + public static final String DONOR_AGENCY_CODE = "Donor Agency Code"; + public static final String EXCHANGE_RATE = "Exchange Rate"; + public static final String RESPONSIBLE_ORGANIZATION = "Responsible Organization"; + public static final String RESPONSIBLE_ORGANIZATION_CODE = "Responsible Organization Code"; + public static final String EXECUTING_AGENCY = "Executing Agency"; + public static final String IMPLEMENTING_AGENCY = "Implementing Agency"; + public static final String BENEFICIARY_AGENCY = "Beneficiary Agency"; + + public static final String ACTUAL_DISBURSEMENT = "Actual Disbursement"; + public static final String ACTUAL_COMMITMENT = "Actual Commitment"; + public static final String ACTUAL_EXPENDITURE = "Actual Expenditure"; + public static final String PLANNED_DISBURSEMENT = "Planned Disbursement"; + public static final String PLANNED_COMMITMENT = "Planned Commitment"; + public static final String PLANNED_EXPENDITURE = "Planned Expenditure"; + public static final String TRANSACTION_AMOUNT = "Transaction Amount"; + public static final String MEASURE_TYPE = "Measure Type"; + public static final String TRANSACTION_DATE = "Transaction Date"; + public static final String FINANCING_INSTRUMENT = "Financing Instrument"; + public static final String TYPE_OF_ASSISTANCE = "Type Of Assistance"; + public static final String PRIMARY_SUBSECTOR = "Primary Subsector"; + public static final String SECONDARY_SUBSECTOR = "Secondary Subsector"; + public static final String CURRENCY = "Currency"; + public static final String COMPONENT_NAME = "Component Name"; + public static final String COMPONENT_CODE = "Component Code"; + + public static final String REPORTING_DATE = "Reporting Date"; + public static final String PROJECT_STATUS = "Project Status"; + + // ----- Indicator (M&E) columns ----- + public static final String INDICATOR_NAME = "Indicator Name"; + public static final String PROGRAM_NAME = "Program Name"; + /** Used for project-level location (e.g. Project Location). */ + public static final String LOCATION = "Location"; + /** Used for matching indicator value to activity location; distinct from project Location. */ + public static final String INDICATOR_LOCATION = "Indicator Location"; + public static final String ORIGINAL_BASE_VALUE = "Original Base Value"; + public static final String ORIGINAL_BASE_VALUE_DATE = "Original Base Value Date"; + public static final String REVISED_BASE_VALUE = "Revised Base Value"; + public static final String REVISED_BASE_VALUE_DATE = "Revised Base Value Date"; + public static final String ORIGINAL_TARGET_VALUE = "Original Target Value"; + public static final String ORIGINAL_TARGET_VALUE_DATE = "Original Target Value Date"; + public static final String REVISED_TARGET_VALUE = "Revised Target Value"; + public static final String REVISED_TARGET_VALUE_DATE = "Revised Target Value Date"; + public static final String ACTUAL_VALUE = "Actual Value"; + public static final String ACTUAL_VALUE_DATE = "Actual Value Date"; + public static final String UNIT_OF_MEASURE = "Unit of Measure"; +} diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java index 357fc6c0c79..3f3e4255400 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java @@ -9,10 +9,13 @@ import org.apache.poi.ss.usermodel.DateUtil; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; +import org.dgfoundation.amp.ar.ArConstants; import org.digijava.kernel.ampapi.endpoints.activity.ActivityImportRules; import org.digijava.kernel.ampapi.endpoints.activity.ActivityInterchangeUtils; import org.digijava.kernel.ampapi.endpoints.activity.dto.ActivitySummary; import org.digijava.kernel.ampapi.endpoints.common.JsonApiResponse; +import org.digijava.kernel.ampapi.endpoints.indicator.manager.IndicatorManagerService; +import org.digijava.kernel.ampapi.endpoints.indicator.manager.MEIndicatorDTO; import org.digijava.kernel.persistence.PersistenceManager; import org.digijava.module.aim.action.dataimporter.dbentity.ImportStatus; import org.digijava.module.aim.action.dataimporter.dbentity.ImportedProject; @@ -20,8 +23,14 @@ import org.digijava.module.aim.action.dataimporter.model.*; import org.digijava.module.aim.dbentity.*; import org.digijava.module.aim.util.CurrencyUtil; +import org.digijava.module.aim.util.FeaturesUtil; +import org.digijava.module.aim.util.ProgramUtil; +import org.digijava.module.aim.util.TeamUtil; +import org.digijava.module.categorymanager.dbentity.AmpCategoryClass; import org.digijava.module.categorymanager.dbentity.AmpCategoryValue; import org.digijava.module.categorymanager.util.CategoryConstants; +import org.digijava.module.categorymanager.util.CategoryManagerUtil; +import org.hibernate.Hibernate; import org.hibernate.Query; import org.hibernate.Session; import org.hibernate.type.StringType; @@ -33,8 +42,6 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; -import java.math.BigDecimal; -import java.math.RoundingMode; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; @@ -69,19 +76,19 @@ private static Double parseDouble(String number) { public static Funding setAFundingItemForExcel(Sheet sheet, Map config, Row row, Map.Entry entry, ImportDataModel importDataModel, Session session, Cell cell, boolean commitment, boolean disbursement, boolean expenditure, String adjustmentType, Funding fundingItem, AmpActivityVersion existingActivity) { - int detailColumn = getColumnIndexByName(sheet, getKey(config, "Financing Instrument")); + int detailColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.FINANCING_INSTRUMENT)); String finInstrument = detailColumn >= 0 ? getStringValueFromCell(row.getCell(detailColumn), false) : ""; - detailColumn = getColumnIndexByName(sheet, getKey(config, "Exchange Rate")); + detailColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.EXCHANGE_RATE)); String exchangeRate = detailColumn >= 0 ? getStringValueFromCell(row.getCell(detailColumn), false) : ""; Double exchangeRateValue = !exchangeRate.isEmpty() ? parseDouble(exchangeRate) : Double.valueOf(0.0); - detailColumn = getColumnIndexByName(sheet, getKey(config, "Type Of Assistance")); + detailColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.TYPE_OF_ASSISTANCE)); String typeOfAss = detailColumn >= 0 ? getStringValueFromCell(row.getCell(detailColumn), false) : ""; - int separateFundingDateColumn = getColumnIndexByName(sheet, getKey(config, "Transaction Date")); + int separateFundingDateColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.TRANSACTION_DATE)); String separateFundingDate = separateFundingDateColumn >= 0 ? getDateFromExcel(row, separateFundingDateColumn) : null; - int currencyCodeColumn = getColumnIndexByName(sheet, getKey(config, "Currency")); + int currencyCodeColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.CURRENCY)); String currencyCode = currencyCodeColumn >= 0 ? getStringValueFromCell(row.getCell(currencyCodeColumn), true) : CurrencyUtil.getDefaultCurrency().getCurrencyCode(); if (existingActivity != null) { String existingActivityCurrencyCode = getCurrencyCodeFromExistingImported(existingActivity.getName()); @@ -91,15 +98,15 @@ public static Funding setAFundingItemForExcel(Sheet sheet, Map c } saveCurrencyCode(currencyCode, importDataModel.getProject_title()); Funding funding; - int componentNameColumn = getColumnIndexByName(sheet, getKey(config, "Component Name")); + int componentNameColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.COMPONENT_NAME)); String componentName = componentNameColumn >= 0 ? getStringValueFromCell(row.getCell(componentNameColumn), true) : null; if (importDataModel.getDonor_organization() == null || importDataModel.getDonor_organization().isEmpty()) { - if (!config.containsValue("Donor Agency")) { + if (!config.containsValue(ImporterConstants.DONOR_AGENCY)) { funding = updateFunding(fundingItem, importDataModel, getNumericValueFromCell(cell), entry.getKey(), separateFundingDate, getRandomOrg(session), typeOfAss, finInstrument, commitment, disbursement, expenditure, adjustmentType, currencyCode, componentName, exchangeRateValue); } else { - int columnIndex1 = getColumnIndexByName(sheet, getKey(config, "Donor Agency")); - int donorAgencyCodeColumn = getColumnIndexByName(sheet, getKey(config, "Donor Agency Code")); + int columnIndex1 = getColumnIndexByName(sheet, getKey(config, ImporterConstants.DONOR_AGENCY)); + int donorAgencyCodeColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.DONOR_AGENCY_CODE)); String donorAgencyCode = donorAgencyCodeColumn >= 0 ? getStringValueFromCell(row.getCell(donorAgencyCodeColumn), true) : null; updateOrgs(importDataModel, columnIndex1 >= 0 ? Objects.requireNonNull(getStringValueFromCell(row.getCell(columnIndex1), false)).trim() : "no org", donorAgencyCode, session, "donor"); funding = updateFunding(fundingItem, importDataModel, getNumericValueFromCell(cell), entry.getKey(), separateFundingDate, new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(), typeOfAss, finInstrument, commitment, disbursement, expenditure, adjustmentType, currencyCode, componentName, exchangeRateValue); @@ -114,17 +121,17 @@ public static Funding setAFundingItemForExcel(Sheet sheet, Map c public static Funding setAFundingItemForTxt(Map row, Map config, Map.Entry entry, ImportDataModel importDataModel, Session session, Number value, boolean commitment, boolean disbursement, boolean expenditure, String adjustmentType, Funding fundingItem, AmpActivityVersion existingActivity) { - String finInstrument = row.get(getKey(config, "Financing Instrument")); + String finInstrument = row.get(getKey(config, ImporterConstants.FINANCING_INSTRUMENT)); finInstrument = finInstrument != null ? finInstrument : ""; - String typeOfAss = row.get(getKey(config, "Type Of Assistance")); + String typeOfAss = row.get(getKey(config, ImporterConstants.TYPE_OF_ASSISTANCE)); typeOfAss = typeOfAss != null ? typeOfAss : ""; Funding funding; - String separateFundingDate = row.get(getKey(config, "Transaction Date")); + String separateFundingDate = row.get(getKey(config, ImporterConstants.TRANSACTION_DATE)); separateFundingDate = separateFundingDate != null ? separateFundingDate : ""; - String currencyCode = row.get(getKey(config, "Currency")); + String currencyCode = row.get(getKey(config, ImporterConstants.CURRENCY)); currencyCode = currencyCode != null ? currencyCode : CurrencyUtil.getDefaultCurrency().getCurrencyCode(); if (existingActivity != null) { String existingActivityCurrencyCode = getCurrencyCodeFromExistingImported(existingActivity.getName()); @@ -133,23 +140,23 @@ public static Funding setAFundingItemForTxt(Map row, Map(importDataModel.getDonor_organization()).get(0).getOrganization(), typeOfAss, finInstrument, commitment, disbursement, expenditure, adjustmentType, currencyCode, componentName, exchangeRateValue); @@ -209,8 +216,13 @@ private static String extractDateFromStringCell(Cell cell) { return null; } - if (rawValue.matches("\\d+")) { + if (rawValue.matches("\\d+(\\.0+)?")) { double numericValue = Double.parseDouble(rawValue); + int intVal = (int) numericValue; + // Year-only: whole number in reasonable year range (e.g. 2023 or 2023.0) -> use as calendar year, not Excel serial days + if (intVal >= 1800 && intVal <= 2700 && numericValue == Math.floor(numericValue)) { + return intVal + "-12-31"; + } if (numericValue > 59) { // Excel bug: after 28 Feb 1900, 60+ is valid Date date = DateUtil.getJavaDate(numericValue); return outputFormat.format(date); @@ -271,6 +283,13 @@ private static Session getSession() { private static String getFundingDate(String dateString) { + if (dateString == null || dateString.trim().isEmpty()) { + return LocalDate.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd")); + } + if (dateString != null && dateString.trim().matches("\\d{4}")) { + int year = Integer.parseInt(dateString.trim()); + return LocalDate.of(year, 12, 31).format(DateTimeFormatter.ofPattern("yyyy-MM-dd")); + } LocalDate date = LocalDate.now(); if (isCommonDateFormat(dateString)) { List formatters = Arrays.asList( @@ -326,9 +345,9 @@ private static String formatDateFromDateObject(String date) { // Check if date is in year-only format (e.g., "2024") if (Pattern.matches("\\d{4}", date)) { try { - // Parse the year and create a Date object for January 1 of that year - Date januaryFirst = new SimpleDateFormat("yyyy-MM-dd").parse(date + "-01-01"); - return new SimpleDateFormat("yyyy-MM-dd").format(januaryFirst); // Return as "yyyy-MM-dd" + // Parse the year and create a Date object for December 31 of that year + Date decemberLast = new SimpleDateFormat("yyyy-MM-dd").parse(date + "-12-31"); + return new SimpleDateFormat("yyyy-MM-dd").format(decemberLast); // Return as "yyyy-MM-dd" } catch (Exception e) { logger.info("Error parsing date", e); } @@ -387,6 +406,58 @@ public static K getKey(Map map, V value) { return null; } + /** + * Result of parsing a Measure Type string (e.g. "PC - Planned Commitment"). + * Used to set commitment/disbursement/expenditure and Actual/Planned for funding. + */ + public static class MeasureTypeResult { + public final boolean commitment; + public final boolean disbursement; + public final boolean expenditure; + public final String adjustmentType; // "Actual" or "Planned" + + public MeasureTypeResult(boolean commitment, boolean disbursement, boolean expenditure, String adjustmentType) { + this.commitment = commitment; + this.disbursement = disbursement; + this.expenditure = expenditure; + this.adjustmentType = adjustmentType; + } + } + + /** + * Parses a Measure Type value from the template (e.g. "PC - Planned Commitment", "AC", or "Actual Commitment"). + * @return MeasureTypeResult or null if not recognized + */ + public static MeasureTypeResult parseMeasureType(String value) { + if (value == null) return null; + String s = value.trim(); + if (s.isEmpty()) return null; + // AC / PC / AD / PD / AE / PE + if (s.equalsIgnoreCase("AC")) return new MeasureTypeResult(true, false, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (s.equalsIgnoreCase("PC")) return new MeasureTypeResult(true, false, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + if (s.equalsIgnoreCase("AD")) return new MeasureTypeResult(false, true, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (s.equalsIgnoreCase("PD")) return new MeasureTypeResult(false, true, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + if (s.equalsIgnoreCase("AE")) return new MeasureTypeResult(false, false, true, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (s.equalsIgnoreCase("PE")) return new MeasureTypeResult(false, false, true, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + // Full form "PC - Planned Commitment" or label only "Planned Commitment" + if (s.contains(" - ")) { + String code = s.substring(0, s.indexOf(" - ")).trim(); + if (code.equalsIgnoreCase("AC")) return new MeasureTypeResult(true, false, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (code.equalsIgnoreCase("PC")) return new MeasureTypeResult(true, false, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + if (code.equalsIgnoreCase("AD")) return new MeasureTypeResult(false, true, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (code.equalsIgnoreCase("PD")) return new MeasureTypeResult(false, true, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + if (code.equalsIgnoreCase("AE")) return new MeasureTypeResult(false, false, true, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (code.equalsIgnoreCase("PE")) return new MeasureTypeResult(false, false, true, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + } + if (s.equalsIgnoreCase(ImporterConstants.ACTUAL_COMMITMENT)) return new MeasureTypeResult(true, false, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (s.equalsIgnoreCase(ImporterConstants.PLANNED_COMMITMENT)) return new MeasureTypeResult(true, false, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + if (s.equalsIgnoreCase(ImporterConstants.ACTUAL_DISBURSEMENT)) return new MeasureTypeResult(false, true, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (s.equalsIgnoreCase(ImporterConstants.PLANNED_DISBURSEMENT)) return new MeasureTypeResult(false, true, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + if (s.equalsIgnoreCase(ImporterConstants.ACTUAL_EXPENDITURE)) return new MeasureTypeResult(false, false, true, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (s.equalsIgnoreCase(ImporterConstants.PLANNED_EXPENDITURE)) return new MeasureTypeResult(false, false, true, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + return null; + } + public static String findYearSubstring(String text) { Pattern pattern = Pattern.compile("(?:19|20)\\d{2}"); Matcher matcher = pattern.matcher(text); @@ -443,12 +514,12 @@ private static Funding updateFunding(Funding fundingItem, ImportDataModel import fundingDate = getFundingDate(separateFundingDate); } else { yearString = findYearSubstring(separateFundingDate); - fundingDate = yearString != null ? getFundingDate(yearString) : getFundingDate("2000"); + fundingDate = yearString != null ? getFundingDate(yearString) : getFundingDate(null); } } else { yearString = findYearSubstring(columnHeaderContainingYear); - fundingDate = yearString != null ? getFundingDate(yearString) : getFundingDate("2000"); + fundingDate = yearString != null ? getFundingDate(yearString) : getFundingDate(null); } @@ -463,20 +534,22 @@ private static Funding updateFunding(Funding fundingItem, ImportDataModel import transaction.setTransaction_amount(amount != null ? amount.doubleValue() : 0.0); transaction.setTransaction_date(fundingDate); transaction.setFixed_exchange_rate(exchangeRate); + + // Check for duplicate transactions by currency, amount, and date before adding if (commitment) { - fundingItem.getCommitments().add(transaction); + if (!transactionExists(fundingItem.getCommitments(), transaction)) { + fundingItem.getCommitments().add(transaction); + } } if (disbursement) { - fundingItem.getDisbursements().add(transaction); + if (!transactionExists(fundingItem.getDisbursements(), transaction)) { + fundingItem.getDisbursements().add(transaction); + } } if (expenditure) { - if (transaction.getTransaction_amount() == 0) { - transaction.setTransaction_amount(-1); + if (!transactionExists(fundingItem.getExpenditures(), transaction)) { + fundingItem.getExpenditures().add(transaction); } - if (transaction.getTransaction_amount() > 0) { - transaction.setTransaction_amount(-transaction.getTransaction_amount()); - } - fundingItem.getCommitments().add(transaction); } @@ -489,6 +562,24 @@ private static Funding updateFunding(Funding fundingItem, ImportDataModel import return fundingItem; } + /** + * Check if a transaction with the same currency, amount, and date already exists in the list + * @param transactions List of existing transactions + * @param newTransaction Transaction to check for duplicates + * @return true if a duplicate exists, false otherwise + */ + private static boolean transactionExists(List transactions, Transaction newTransaction) { + if (transactions == null || newTransaction == null) { + return false; + } + + return transactions.stream().anyMatch(existing -> + existing.getCurrency() != null && existing.getCurrency().equals(newTransaction.getCurrency()) && + Double.compare(existing.getTransaction_amount(), newTransaction.getTransaction_amount()) == 0 && + existing.getTransaction_date() != null && existing.getTransaction_date().equals(newTransaction.getTransaction_date()) + ); + } + private static Long getOrganizationRole(Session session) { if (ConstantsMap.containsKey("orgRole")) { @@ -563,6 +654,64 @@ private static Long getCategoryValue(String constantKey, String categoryKey, Str return categoryId; } + /** + * Resolves activity (project) status by value: looks up existing category value for ACTIVITY_STATUS_KEY; + * if not found in DB, creates a new category value and returns its id. + * @param statusValue value from the file (e.g. "Ongoing", "Completed") + * @param session current session (used for create and flush) + * @return category value id, or null if statusValue is null/empty + */ + public static Long getOrCreateActivityStatusCategoryValue(String statusValue, Session session) { + if (statusValue == null || statusValue.trim().isEmpty()) return null; + String trimmed = statusValue.trim(); + String cacheKey = "statusId_" + trimmed; + if (ConstantsMap.containsKey(cacheKey)) { + return ConstantsMap.get(cacheKey); + } + if (!session.isOpen()) { + session = PersistenceManager.getRequestDBSession(); + } + String hql = "SELECT s FROM " + AmpCategoryValue.class.getName() + " s JOIN s.ampCategoryClass c WHERE c.keyName = :categoryKey"; + Query query = session.createQuery(hql); + query.setParameter("categoryKey", CategoryConstants.ACTIVITY_STATUS_KEY); + @SuppressWarnings("unchecked") + List values = (List) query.list(); + if (values != null) { + for (AmpCategoryValue cv : values) { + if (cv.getValue() != null && cv.getValue().equalsIgnoreCase(trimmed)) { + Long id = cv.getId(); + ConstantsMap.put(cacheKey, id); + return id; + } + } + } + AmpCategoryClass categoryClass = CategoryManagerUtil.loadAmpCategoryClassByKey(CategoryConstants.ACTIVITY_STATUS_KEY); + if (categoryClass == null) { + logger.warn("Activity status category class not found; cannot create value: " + trimmed); + return null; + } + try { + AmpCategoryValue newValue = new AmpCategoryValue(); + newValue.setValue(trimmed); + newValue.setAmpCategoryClass(categoryClass); + if (categoryClass.getPossibleValues() == null) { + categoryClass.setPossibleValues(new java.util.ArrayList<>()); + } + newValue.setIndex(categoryClass.getPossibleValues().size()); + session.save(newValue); + session.flush(); + Long id = newValue.getId(); + if (id != null) { + ConstantsMap.put(cacheKey, id); + logger.info("Created new activity status category value: " + trimmed + " (id=" + id + ")"); + return id; + } + } catch (Exception e) { + logger.warn("Failed to create activity status value: " + trimmed, e); + } + return null; + } + public static AmpActivityVersion existingActivity(String projectTitle, String projectCode, Session session) { if ((projectTitle == null || projectTitle.trim().isEmpty()) && (projectCode == null || projectCode.trim().isEmpty())) { @@ -571,43 +720,106 @@ public static AmpActivityVersion existingActivity(String projectTitle, String pr if (!session.isOpen()) { session = PersistenceManager.getRequestDBSession(); } - String hql = "SELECT a FROM " + AmpActivityVersion.class.getName() + " a " + - "WHERE a.name = :name"; - Query query = session.createQuery(hql); - query.setCacheable(true); - query.setParameter("name", projectTitle, StringType.INSTANCE); -// query.setString("projectCode", projectCode); - List ampActivityVersions = query.list(); - return !ampActivityVersions.isEmpty() ? ampActivityVersions.get(ampActivityVersions.size() - 1) : null; + // Prefer project code if provided + if (projectCode != null && !projectCode.trim().isEmpty()) { + String hqlByCode = "SELECT a FROM " + AmpActivityVersion.class.getName() + " a LEFT JOIN FETCH a.activityCreator WHERE a.projectCode = :projectCode"; + Query queryByCode = session.createQuery(hqlByCode); + queryByCode.setCacheable(true); + queryByCode.setParameter("projectCode", projectCode.trim(), StringType.INSTANCE); + List byCode = queryByCode.list(); + if (!byCode.isEmpty()) { + return byCode.get(byCode.size() - 1); + } + } + // Fall back to project title (name) + if (projectTitle != null && !projectTitle.trim().isEmpty()) { + String hql = "SELECT a FROM " + AmpActivityVersion.class.getName() + " a LEFT JOIN FETCH a.activityCreator WHERE a.name = :name"; + Query query = session.createQuery(hql); + query.setCacheable(true); + query.setParameter("name", projectTitle.trim(), StringType.INSTANCE); + List ampActivityVersions = query.list(); + return !ampActivityVersions.isEmpty() ? ampActivityVersions.get(ampActivityVersions.size() - 1) : null; + } + return null; } + /** + * Sets default activity status and approval status on the import model. + * If activity_status is already set (e.g. from Project Status column), it is left unchanged. + */ public static void setStatus(ImportDataModel importDataModel) { - Long statusId = getCategoryValue("statusId", CategoryConstants.ACTIVITY_STATUS_KEY, ""); - importDataModel.setActivity_status(statusId); + if (importDataModel.getActivity_status() == null) { + Long statusId = getCategoryValue("statusId", CategoryConstants.ACTIVITY_STATUS_KEY, ""); + importDataModel.setActivity_status(statusId); + } importDataModel.setApproval_status(ApprovalStatus.started.getId()); } - public static void importTheData(ImportDataModel importDataModel, Session session, ImportedProject importedProject, String componentName, String componentCode, Long responsibleOrgId, List fundings, AmpActivityVersion existing) throws JsonProcessingException { - if (!session.isOpen()) { + private static final String CREATED_BY_KEY = "created_by"; + + /** + * Ensures created_by in the activity map is set to a valid team member id when null, + * so the activity API validator does not reject with "(Invalid field value) created_by". + * For new activities uses current user; for updates uses existing activity's creator only + * when that creator is present (never overwrite with current user for existing activities). + */ + private static void ensureCreatedBySet(Map map, AmpActivityVersion existing) { + if (existing != null) { + AmpTeamMember creator = existing.getActivityCreator(); + if (creator == null) { + // Existing activity has no creator (legacy); API expects null. + map.put(CREATED_BY_KEY, null); + return; + } + map.put(CREATED_BY_KEY, creator.getAmpTeamMemId()); + return; + } + Object createdBy = map.get(CREATED_BY_KEY); + if (createdBy != null) { + return; + } + AmpTeamMember currentMember = TeamUtil.getCurrentAmpTeamMember(); + if (currentMember != null) { + map.put(CREATED_BY_KEY, currentMember.getAmpTeamMemId()); + } + } + + /** @return activity ID on success, null on skip or failure */ + public static Long importTheData(ImportDataModel importDataModel, Session session, ImportedProject importedProject, String componentName, String componentCode, Long responsibleOrgId, List fundings, Long existingActivityId) throws JsonProcessingException { + if (session == null || !session.isOpen()) { session = PersistenceManager.getRequestDBSession(); } + + // Re-fetch existing activity in this transaction if ID is provided to avoid detached entity issues + AmpActivityVersion existing = null; + if (existingActivityId != null) { + existing = session.get(AmpActivityVersion.class, existingActivityId); + } ActivityImportRules rules = new ActivityImportRules(true, false, true); ObjectMapper objectMapper = new ObjectMapper(); objectMapper.configure(ESCAPE_NON_ASCII, false); // Disable escaping of non-ASCII characters during serialization objectMapper.configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true); + normalizeLocationPercentages(importDataModel); Map map = objectMapper .convertValue(importDataModel, new TypeReference>() { }); + // Remove null values and "null" strings from the map to avoid API validation errors + map.entrySet().removeIf(entry -> entry.getValue() == null || "null".equals(String.valueOf(entry.getValue()))); + + // Do not send indicators in the payload so activity/update does not replace or clear existing indicators. + // Indicator data is appended separately in addIndicatorDataToActivity. + map.remove("indicators"); JsonApiResponse response; logger.info("Data model object: " + importDataModel); if (importDataModel.getProject_title().trim().isEmpty() && importDataModel.getProject_code().trim().isEmpty()) { logger.info("Project title and code are empty. Skipping import"); importedProject.setImportStatus(ImportStatus.SKIPPED); - return; + return null; } if (existing == null) { + ensureCreatedBySet(map, null); logger.info("New activity"); importedProject.setNewProject(true); response = ActivityInterchangeUtils.importActivity(map, false, rules, "activity/new"); @@ -616,103 +828,223 @@ public static void importTheData(ImportDataModel importDataModel, Session sessio importedProject.setNewProject(false); importDataModel.setInternal_id(existing.getAmpActivityId()); importDataModel.setAmp_id(existing.getAmpId()); - ActivityGroup activityGroup = new ActivityGroup(); - activityGroup.setVersion(existing.getAmpActivityGroup().getVersion()); - importDataModel.setActivity_group(activityGroup); - importDataModel.setProject_title(existing.getName()); - importDataModel.setProject_code(!Objects.equals(importDataModel.getProject_code(), "") ? importDataModel.getProject_code() : existing.getProjectCode()); + // Only set activity group if it exists and has the data we need + if (existing.getAmpActivityGroup() != null) { + ActivityGroup activityGroup = new ActivityGroup(); + activityGroup.setVersion(existing.getAmpActivityGroup().getVersion()); + importDataModel.setActivity_group(activityGroup); + } + importDataModel.setProject_title(existing.getName() != null ? existing.getName() : ""); + importDataModel.setProject_code(!Objects.equals(importDataModel.getProject_code(), "") ? importDataModel.getProject_code() : (existing.getProjectCode() != null ? existing.getProjectCode() : "")); updateFundingOrgsAndSectorsWithAlreadyExisting(existing, importDataModel); + // Merge existing activity locations into payload so we only add (row + existing), never remove + mergeExistingActivityLocationsIntoImport(existing, importDataModel); + ensureImplementationLevelWhenHasLocations(importDataModel, session); + normalizeLocationPercentages(importDataModel); map = objectMapper .convertValue(importDataModel, new TypeReference>() { }); - response = ActivityInterchangeUtils.importActivity(map, true, rules, "activity/update"); + // Remove null values and "null" strings from the map to avoid API validation errors + map.entrySet().removeIf(entry -> entry.getValue() == null || "null".equals(String.valueOf(entry.getValue()))); + + map.remove("indicators"); // preserve existing indicators; we append in addIndicatorDataToActivity + // Do not replace programs; avoids StaleStateException when deleting AMP_ACTIVITY_PROGRAM rows + map.remove("national_plan_objective"); + map.remove("primary_programs"); + map.remove("secondary_programs"); + map.remove("tertiary_programs"); + // Avoid triggering merge of contacts/documents that may reference deleted rows (ObjectNotFoundException) + map.remove("activity_contacts"); + map.remove("activityContacts"); + map.remove("donor_contact_information"); + map.remove("project_coordinator_contact_information"); + map.remove("sector_ministry_contact_information"); + map.remove("mofed_contact_information"); + map.remove("implementing_executing_agency_contact_information"); + evictActivityFromSecondLevelCache(existing.getAmpActivityId()); + ensureCreatedBySet(map, existing); + try { + response = ActivityInterchangeUtils.importActivity(map, true, rules, "activity/update"); + } catch (Exception e) { + logger.error("Activity import failed for row", e); + importedProject.setImportStatus(ImportStatus.FAILED); + Map> errMap = new LinkedHashMap<>(); + errMap.put("1", Collections.singletonList("Internal Error : [" + (e.getMessage() != null ? e.getMessage() : "Activity import failed") + "]")); + response = new JsonApiResponse<>(errMap, null, null, null); + } } + Long activityId = null; if (response != null) { if (!response.getErrors().isEmpty()) { importedProject.setImportStatus(ImportStatus.FAILED); } else { importedProject.setImportStatus(ImportStatus.SUCCESS); + activityId = existing != null ? existing.getAmpActivityId() : (Long) response.getContent().getAmpActivityId(); logger.info("Successfully imported the project. Now adding component if present"); logger.info("--------------------------------"); logger.info("Component name at start: " + componentName); if (componentName != null && !componentName.isEmpty()) { addComponentsAndProjectCode(response, componentName, componentCode, responsibleOrgId, fundings, importDataModel.getProject_code()); } -// logger.info("Updating expenditures ................"); -// updateExpendituresIfAny(response); - } } String resp = objectMapper.writeValueAsString(response); importedProject.setImportResponse(resp); - if (!session.isOpen()) { - session = PersistenceManager.getRequestDBSession(); + try { + PersistenceManager.doInTransaction(s -> { + s.saveOrUpdate(importedProject); + s.flush(); + }); + } catch (Exception e) { + logger.warn("Could not save import status for imported project (response already set): {}", e.getMessage()); } - session.saveOrUpdate(importedProject); - session.flush(); logger.info("Imported project: " + importedProject); + return activityId; } private static void updateFundingOrgsAndSectorsWithAlreadyExisting(AmpActivityVersion ampActivityVersion, ImportDataModel importDataModel) { if (ampActivityVersion.getFunding() != null) { + Hibernate.initialize(ampActivityVersion.getFunding()); Long adjType = getCategoryValue("adjustmentType", CategoryConstants.ADJUSTMENT_TYPE_KEY, ""); Long assType = getCategoryValue("assistanceType", CategoryConstants.TYPE_OF_ASSISTENCE_KEY, ""); Long finInstrument = getCategoryValue("finInstrument", CategoryConstants.FINANCING_INSTRUMENT_KEY, ""); + if (importDataModel.getFundings() == null) importDataModel.setFundings(new HashSet<>()); for (AmpFunding ampFunding : ampActivityVersion.getFunding()) { Funding funding = new Funding(); + if (ampFunding.getAmpFundingId() != null) funding.setFunding_id(ampFunding.getAmpFundingId()); funding.setDonor_organization_id(ampFunding.getAmpDonorOrgId().getAmpOrgId()); funding.setType_of_assistance(ampFunding.getTypeOfAssistance() != null ? ampFunding.getTypeOfAssistance().getId() : assType); funding.setFinancing_instrument(ampFunding.getFinancingInstrument() != null ? ampFunding.getFinancingInstrument().getId() : finInstrument); funding.setSource_role(ampFunding.getSourceRole().getAmpRoleId()); - for (AmpFundingDetail ampFundingDetail : ampFunding.getFundingDetails()) { - Transaction transaction = new Transaction(); - transaction.setCurrency(ampFundingDetail.getAmpCurrencyId().getAmpCurrencyId()); - transaction.setAdjustment_type(ampFundingDetail.getAdjustmentType() != null ? ampFundingDetail.getAdjustmentType().getId() : adjType); - transaction.setTransaction_amount(ampFundingDetail.getTransactionAmount()); - if (ampFundingDetail.getTransactionDate() != null) { - - transaction.setTransaction_date(getFundingDate(ampFundingDetail.getTransactionDate().toInstant() - .atZone(ZoneId.systemDefault()) - .toLocalDate().toString())); - } - transaction.setFixed_exchange_rate(ampFundingDetail.getFixedExchangeRate()); - if (ampFundingDetail.getTransactionType() == 0) { - funding.getCommitments().add(transaction); - } else if (ampFundingDetail.getTransactionType() == 1) { - funding.getDisbursements().add(transaction); + if (ampFunding.getFundingDetails() != null) { + Hibernate.initialize(ampFunding.getFundingDetails()); + for (AmpFundingDetail ampFundingDetail : ampFunding.getFundingDetails()) { + Transaction transaction = new Transaction(); + if (ampFundingDetail.getAmpFundDetailId() != null) transaction.setTransaction_id(ampFundingDetail.getAmpFundDetailId()); + transaction.setCurrency(ampFundingDetail.getAmpCurrencyId().getAmpCurrencyId()); + transaction.setAdjustment_type(ampFundingDetail.getAdjustmentType() != null ? ampFundingDetail.getAdjustmentType().getId() : adjType); + transaction.setTransaction_amount(ampFundingDetail.getTransactionAmount()); + if (ampFundingDetail.getTransactionDate() != null) { + transaction.setTransaction_date(getFundingDate(ampFundingDetail.getTransactionDate().toInstant() + .atZone(ZoneId.systemDefault()) + .toLocalDate().toString())); + } + transaction.setFixed_exchange_rate(ampFundingDetail.getFixedExchangeRate()); + if (ampFundingDetail.getTransactionType() == 0) { + funding.getCommitments().add(transaction); + } else if (ampFundingDetail.getTransactionType() == 1) { + funding.getDisbursements().add(transaction); + } } } - + importDataModel.getFundings().add(funding); } } if (ampActivityVersion.getOrgrole() != null && !ampActivityVersion.getOrgrole().isEmpty()) { for (AmpOrgRole ampOrgRole : ampActivityVersion.getOrgrole()) { - if (ampOrgRole.getRole().getRoleCode().equalsIgnoreCase("DN")) { - createDonorOrg(importDataModel,ampOrgRole.getOrganisation().getAmpOrgId()); - } else if (ampOrgRole.getRole().getRoleCode().equalsIgnoreCase("EA")) { + if (ampOrgRole.getRole() == null) continue; + String roleCode = ampOrgRole.getRole().getRoleCode(); + if (roleCode == null) continue; + if (roleCode.equalsIgnoreCase("DN")) { + createDonorOrg(importDataModel, ampOrgRole.getOrganisation().getAmpOrgId(), ampOrgRole.getAmpOrgRoleId()); + } else if (roleCode.equalsIgnoreCase("EA")) { Organization responsibleOrg = new Organization(); responsibleOrg.setOrganization(ampOrgRole.getOrganisation().getAmpOrgId()); + if (ampOrgRole.getAmpOrgRoleId() != null) responsibleOrg.setId(ampOrgRole.getAmpOrgRoleId()); importDataModel.getResponsible_organization().add(responsibleOrg); - } else if (ampOrgRole.getRole().getRoleCode().equalsIgnoreCase("BA")) { + } else if (roleCode.equalsIgnoreCase("BA")) { Organization beneficiaryAgency = new Organization(); beneficiaryAgency.setOrganization(ampOrgRole.getOrganisation().getAmpOrgId()); + if (ampOrgRole.getAmpOrgRoleId() != null) beneficiaryAgency.setId(ampOrgRole.getAmpOrgRoleId()); importDataModel.getBeneficiary_agency().add(beneficiaryAgency); - } } } - if (ampActivityVersion.getSectors() != null && !ampActivityVersion.getSectors().isEmpty()) { + Hibernate.initialize(ampActivityVersion.getSectors()); for (AmpActivitySector ampActivitySector : ampActivityVersion.getSectors()) { - createSector(importDataModel,ampActivitySector.getClassificationConfig().getName().equalsIgnoreCase("primary"),ampActivitySector.getSectorId().getAmpSectorId()); + if (ampActivitySector.getSectorId() == null) continue; + boolean primary = ampActivitySector.getClassificationConfig() != null && "primary".equalsIgnoreCase(ampActivitySector.getClassificationConfig().getName()); + createSector(importDataModel, primary, ampActivitySector.getSectorId().getAmpSectorId(), ampActivitySector.getAmpActivitySectorId()); + } + } + } + + /** + * For an existing activity, merges its current locations into the import payload so we only add locations + * (row locations + existing), never remove. Any existing activity location not already in importDataModel + * is added. This avoids activity/update deleting locations (e.g. those referenced by indicator connections). + */ + private static void mergeExistingActivityLocationsIntoImport(AmpActivityVersion existing, ImportDataModel importDataModel) { + if (existing == null || importDataModel == null) return; + if (existing.getLocations() == null) return; + Hibernate.initialize(existing.getLocations()); + Set alreadyInImport = new HashSet<>(); + if (importDataModel.getLocations() != null) { + for (Location loc : importDataModel.getLocations()) { + if (loc != null && loc.getLocation() != null) alreadyInImport.add(loc.getLocation()); + } + } + for (AmpActivityLocation aal : existing.getLocations()) { + AmpCategoryValueLocations loc = aal.getLocation(); + if (loc == null) continue; + Long locId = loc.getId(); + if (locId == null || alreadyInImport.contains(locId)) continue; + if (importDataModel.getLocations() == null) importDataModel.setLocations(new HashSet<>()); + double pct = aal.getLocationPercentage() != null ? aal.getLocationPercentage().doubleValue() : 100.0; + // Include aal.getId() (amp_activity_location_id) so the API matches and keeps this row; otherwise removeByIdExcept drops it and Hibernate deletes it (FK violation if referenced by amp_indicator_connection). + Long aalId = aal.getId(); + importDataModel.getLocations().add(aalId != null ? new Location(aalId, locId, pct) : new Location(locId, pct)); + alreadyInImport.add(locId); + } + } + + /** + * Scales location percentages so they sum to 100, as required by activity validation. + * If there are no locations or sum is 0, does nothing. + */ + private static void normalizeLocationPercentages(ImportDataModel importDataModel) { + if (importDataModel == null || importDataModel.getLocations() == null || importDataModel.getLocations().isEmpty()) + return; + Set locs = importDataModel.getLocations(); + double sum = 0; + for (Location loc : locs) { + Double pct = loc.getLocation_percentage(); + sum += (pct != null ? pct : 0); + } + if (sum <= 0) return; + if (Math.abs(sum - 100.0) < 0.001) return; // already 100 + List list = new ArrayList<>(locs); + double scale = 100.0 / sum; + double running = 0; + for (int i = 0; i < list.size(); i++) { + Location loc = list.get(i); + double v; + if (i == list.size() - 1) { + v = 100.0 - running; // last one gets remainder so total is exactly 100 + } else { + Double pct = loc.getLocation_percentage(); + v = (pct != null ? pct : 0) * scale; + running += v; } + loc.setLocation_percentage(v); } } + /** + * When the payload has locations, implementation level is required. Sets default if missing (e.g. after merging locations for existing activity). + */ + private static void ensureImplementationLevelWhenHasLocations(ImportDataModel importDataModel, Session session) { + if (importDataModel == null || importDataModel.getLocations() == null || importDataModel.getLocations().isEmpty()) + return; + if (importDataModel.getImplementation_level() != null) return; + updateImpLevels(importDataModel, session); + } + static void updateExpendituresIfAny(JsonApiResponse response) { Long activityId = (Long) response.getContent().getAmpActivityId(); Session session = PersistenceManager.getRequestDBSession(); @@ -989,45 +1321,110 @@ public static void updateSectors(ImportDataModel importDataModel, String name, S } - public static void updateLocations(ImportDataModel importDataModel, String locationName, Session session) { - logger.info("Updating locations"); - - if (ConstantsMap.containsKey("location_" + locationName)) { - Long location = ConstantsMap.get("location_" + locationName); - logger.info("In cache... location " + "location_" + locationName + ":" + location); - importDataModel.getLocations().add(new Location(location, 100.00)); - - } else { - if (!session.isOpen()) { - session = PersistenceManager.getRequestDBSession(); - } - - session.doWork(connection -> { - String query = "SELECT acvl.id AS location_id FROM amp_category_value_location acvl WHERE LOWER(acvl.location_name) = LOWER(?)"; - try (PreparedStatement statement = connection.prepareStatement(query)) { - statement.setString(1, locationName); + /** + * Splits a comma- or semicolon-separated string into non-empty trimmed parts. + */ + private static List splitLocationNames(String locationNames) { + if (locationNames == null || locationNames.trim().isEmpty()) return Collections.emptyList(); + List result = new ArrayList<>(); + for (String part : locationNames.split("[,;]")) { + String trimmed = part.trim(); + if (!trimmed.isEmpty()) result.add(trimmed); + } + return result; + } - try (ResultSet resultSet = statement.executeQuery()) { - while (resultSet.next()) { - Long location = resultSet.getLong("location_id"); - logger.info("Location:" + location); - importDataModel.getLocations().add(new Location(location, 100.00)); -// importDataModel.setImplementation_location(location); - ConstantsMap.put("location_" + locationName, location); - } - } + public static void updateLocations(ImportDataModel importDataModel, String locationNames, Session session) { + logger.info("Updating locations"); + if (locationNames == null || locationNames.trim().isEmpty()) return; + for (String locationName : splitLocationNames(locationNames)) { + if (ConstantsMap.containsKey("location_" + locationName)) { + Long location = ConstantsMap.get("location_" + locationName); + logger.info("In cache... location " + "location_" + locationName + ":" + location); + importDataModel.getLocations().add(new Location(location, 100.00)); - } catch (SQLException e) { - logger.error("Error getting locations", e); + } else { + if (!session.isOpen()) { + session = PersistenceManager.getRequestDBSession(); } - }); + final String locationNameFinal = locationName; + session.doWork(connection -> { + String query = "SELECT acvl.id AS location_id FROM amp_category_value_location acvl WHERE LOWER(acvl.location_name) = LOWER(?)"; + try (PreparedStatement statement = connection.prepareStatement(query)) { + statement.setString(1, locationNameFinal); + + try (ResultSet resultSet = statement.executeQuery()) { + while (resultSet.next()) { + Long location = resultSet.getLong("location_id"); + logger.info("Location:" + location); + importDataModel.getLocations().add(new Location(location, 100.00)); + ConstantsMap.put("location_" + locationNameFinal, location); + } + } + } catch (SQLException e) { + logger.error("Error getting locations", e); + } + }); + } } - updateImpLevels(importDataModel,session); - + updateImpLevels(importDataModel, session); + } + /** + * Ensures the activity has an activity location for the given location name (for indicator location). + * If the location is not already on the activity, resolves it by name and adds it. + * @return the AmpActivityLocation for the name, or null if the location name cannot be resolved + */ + private static AmpActivityLocation getOrAddActivityLocationForName(AmpActivityVersion activity, String locationName, Session session) { + if (activity == null || locationName == null || locationName.trim().isEmpty()) return null; + locationName = locationName.trim(); + if (ConstantsMap.containsKey("location_" + locationName)) { + Long locationId = ConstantsMap.get("location_" + locationName); + AmpCategoryValueLocations loc = session.get(AmpCategoryValueLocations.class, locationId); + if (loc == null) return null; + AmpActivityLocation aal = new AmpActivityLocation(); + aal.setActivity(activity); + aal.setLocation(loc); + aal.setLocationPercentage(100f); + if (activity.getLocations() == null) activity.setLocations(new HashSet<>()); + activity.getLocations().add(aal); + session.save(aal); + session.flush(); + return aal; + } + if (!session.isOpen()) { + session = PersistenceManager.getRequestDBSession(); + } + final String locationNameFinal = locationName; + final Long[] foundId = new Long[1]; + session.doWork(connection -> { + String query = "SELECT acvl.id AS location_id FROM amp_category_value_location acvl WHERE LOWER(acvl.location_name) = LOWER(?)"; + try (PreparedStatement statement = connection.prepareStatement(query)) { + statement.setString(1, locationNameFinal); + try (ResultSet resultSet = statement.executeQuery()) { + if (resultSet.next()) { + foundId[0] = resultSet.getLong("location_id"); + ConstantsMap.put("location_" + locationNameFinal, foundId[0]); + } + } + } catch (SQLException e) { + logger.error("Error resolving location by name: " + locationNameFinal, e); + } + }); + if (foundId[0] == null) return null; + AmpCategoryValueLocations loc = session.get(AmpCategoryValueLocations.class, foundId[0]); + if (loc == null) return null; + AmpActivityLocation aal = new AmpActivityLocation(); + aal.setActivity(activity); + aal.setLocation(loc); + aal.setLocationPercentage(100f); + if (activity.getLocations() == null) activity.setLocations(new HashSet<>()); + activity.getLocations().add(aal); + session.save(aal); + return aal; } public static void updateImpLevels(ImportDataModel importDataModel, Session session) @@ -1063,23 +1460,25 @@ public static void updateImpLevels(ImportDataModel importDataModel, Session sess } private static void createSector(ImportDataModel importDataModel, boolean primary, Long ampSectorId) { - Sector sector1 = new Sector(); + createSector(importDataModel, primary, ampSectorId, null); + } + private static void createSector(ImportDataModel importDataModel, boolean primary, Long ampSectorId, Long sectorPkId) { + Sector sector1 = new Sector(); sector1.setSector(ampSectorId); + if (sectorPkId != null) sector1.setId(sectorPkId); if (primary) { importDataModel.getPrimary_sectors().add(sector1); Map percentages = divide100(importDataModel.getPrimary_sectors().size()); - int index=0; + int index = 0; for (Sector sec : importDataModel.getPrimary_sectors()) { sec.setSector_percentage(percentages.get(index)); index++; } - } - else - { + } else { importDataModel.getSecondary_sectors().add(sector1); Map percentages = divide100(importDataModel.getSecondary_sectors().size()); - int index=0; + int index = 0; for (Sector sec : importDataModel.getSecondary_sectors()) { sec.setSector_percentage(percentages.get(index)); index++; @@ -1195,11 +1594,16 @@ public static Map divide100(int n) { } private static void createDonorOrg(ImportDataModel importDataModel, Long orgId) { + createDonorOrg(importDataModel, orgId, null); + } + + private static void createDonorOrg(ImportDataModel importDataModel, Long orgId, Long orgRoleId) { DonorOrganization donorOrganization = new DonorOrganization(); donorOrganization.setOrganization(orgId); + if (orgRoleId != null) donorOrganization.setId(orgRoleId); importDataModel.getDonor_organization().add(donorOrganization); Map percentages = divide100(importDataModel.getDonor_organization().size()); - int index=0; + int index = 0; for (DonorOrganization donorOrganization1 : importDataModel.getDonor_organization()) { donorOrganization1.setPercentage(percentages.get(index)); index++; @@ -1223,4 +1627,573 @@ public static int getColumnIndexByName(Sheet sheet, String columnName) { } } + + /** Parse date from Excel cell or string; returns today if null/empty/invalid. */ + public static Date parseDateDefaultToday(Row row, Sheet sheet, Map config, String columnName) { + String key = getKey(config, columnName); + if (key == null) return Date.from(LocalDate.now().atStartOfDay(ZoneId.systemDefault()).toInstant()); + int col = getColumnIndexByName(sheet, key); + if (col < 0) return Date.from(LocalDate.now().atStartOfDay(ZoneId.systemDefault()).toInstant()); + Cell cell = row.getCell(col); + if (cell == null) return Date.from(LocalDate.now().atStartOfDay(ZoneId.systemDefault()).toInstant()); + String dateStr = extractDateFromStringCell(cell); + if (dateStr == null || dateStr.isEmpty()) return Date.from(LocalDate.now().atStartOfDay(ZoneId.systemDefault()).toInstant()); + try { + return java.sql.Date.valueOf(dateStr); + } catch (Exception e) { + return Date.from(LocalDate.now().atStartOfDay(ZoneId.systemDefault()).toInstant()); + } + } + + /** Add indicator data to an activity from the current row. Called after importTheData when indicator columns are mapped. */ + public static void addIndicatorDataToActivity(Long activityId, Row row, Sheet sheet, Map config, Session session) { + logger.info("addIndicatorDataToActivity: activityId={}, row={}", activityId, row != null ? row.getRowNum() : null); + if (activityId == null || config == null || row == null || sheet == null) { + logger.info("addIndicatorDataToActivity: skipping - activityId, config, row or sheet is null"); + return; + } + // importTheData runs inside ActivityGatekeeper.doWithLock which commits and closes the session; use a fresh one if closed + if (session == null || !session.isOpen()) { + session = PersistenceManager.getRequestDBSession(); + logger.info("addIndicatorDataToActivity: obtained fresh session"); + } + String locationConfigKey = getKey(config, ImporterConstants.INDICATOR_LOCATION) != null + ? ImporterConstants.INDICATOR_LOCATION + : ImporterConstants.LOCATION; + if (getKey(config, ImporterConstants.INDICATOR_NAME) == null || getKey(config, locationConfigKey) == null || getKey(config, ImporterConstants.ACTUAL_VALUE) == null) { + logger.info("addIndicatorDataToActivity: skipping - missing config for indicator name, location or actual value"); + return; + } + String indicatorName = getCellValueByConfig(row, sheet, config, ImporterConstants.INDICATOR_NAME); + String locationNamesStr = getCellValueByConfig(row, sheet, config, locationConfigKey); + if (indicatorName == null || indicatorName.trim().isEmpty() || locationNamesStr == null || locationNamesStr.trim().isEmpty()) { + logger.info("addIndicatorDataToActivity: skipping - indicatorName or locationNamesStr empty (indicatorName='{}', locationNamesStr='{}')", indicatorName, locationNamesStr); + return; + } + indicatorName = indicatorName.trim(); + List locationNames = splitLocationNames(locationNamesStr); + logger.info("addIndicatorDataToActivity: indicator='{}', locations(count={}): {}", indicatorName, locationNames.size(), locationNames); + if (locationNames.isEmpty()) { + logger.debug("addIndicatorDataToActivity: no location names after split"); + return; + } + + String programName = getCellValueByConfig(row, sheet, config, ImporterConstants.PROGRAM_NAME); + if (programName != null) programName = programName.trim(); + + AmpTheme programTheme = null; + if (programName != null && !programName.isEmpty()) { + try { + programTheme = getOrCreateProgramByName(programName, session); + } catch (Exception e) { + logger.error("Could not resolve or create program by name: " + programName, e); + } + } + + IndicatorManagerService indicatorService = new IndicatorManagerService(); + MEIndicatorDTO indicatorDto = indicatorService.getMeIndicatorByNameAndProgramNameOptional(indicatorName, (programName == null || programName.isEmpty()) ? null : programName); + AmpIndicator indicator; + if (indicatorDto == null) { + logger.info("addIndicatorDataToActivity: creating new indicator '{}' (program={})", indicatorName, programName); + MEIndicatorDTO createDto = new MEIndicatorDTO(); + createDto.setName(indicatorName); + createDto.setCode(indicatorName + "_" + System.currentTimeMillis()); + createDto.setCreationDate(new Date()); + createDto.setAscending(true); + createDto.setSectorIds(new ArrayList<>()); + if (programTheme != null && programTheme.getAmpThemeId() != null) { + createDto.setProgramId(programTheme.getAmpThemeId()); + } + try { + indicatorDto = indicatorService.createMEIndicator(createDto); + } catch (Exception e) { + logger.error("Failed to create indicator: " + indicatorName, e); + return; + } + indicator = session.get(AmpIndicator.class, indicatorDto.getId()); + logger.info("addIndicatorDataToActivity: created indicator id={}", indicator != null ? indicator.getIndicatorId() : null); + } else { + indicator = session.get(AmpIndicator.class, indicatorDto.getId()); + logger.info("addIndicatorDataToActivity: using existing indicator id={} name='{}'", indicator != null ? indicator.getIndicatorId() : null, indicatorName); + } + if (indicator == null) { + logger.info("addIndicatorDataToActivity: indicator is null after lookup/create"); + return; + } + + AmpActivityVersion activity = session.get(AmpActivityVersion.class, activityId); + if (activity == null) { + logger.info("addIndicatorDataToActivity: activity not found for activityId={}", activityId); + return; + } + // Force-load indicators so we append to existing; avoid replacing due to lazy/uninitialized collection + if (activity.getIndicators() != null) { + Hibernate.initialize(activity.getIndicators()); + } + int indicatorsCountBefore = activity.getIndicators() == null ? 0 : activity.getIndicators().size(); + logger.info("addIndicatorDataToActivity: activityId={} existing indicators count={}", activityId, indicatorsCountBefore); + + if (programTheme != null) { + addProgramToActivityIfMissing(activity, programTheme, session); + } + + AmpIndicatorGlobalValue existingBase = indicator.getBaseValue(); + double origBase = parseDoubleFromConfig(row, sheet, config, ImporterConstants.ORIGINAL_BASE_VALUE); + boolean hasOrigBase = getKey(config, ImporterConstants.ORIGINAL_BASE_VALUE) != null && !Double.isNaN(origBase); + double revBase = parseDoubleFromConfig(row, sheet, config, ImporterConstants.REVISED_BASE_VALUE); + boolean hasRevBase = getKey(config, ImporterConstants.REVISED_BASE_VALUE) != null && !Double.isNaN(revBase); + double origTarget = parseDoubleFromConfig(row, sheet, config, ImporterConstants.ORIGINAL_TARGET_VALUE); + double revTarget = parseDoubleFromConfig(row, sheet, config, ImporterConstants.REVISED_TARGET_VALUE); + String actualValueConfigKey = getKey(config, ImporterConstants.ACTUAL_VALUE); + double actualVal = parseDoubleFromConfig(row, sheet, config, ImporterConstants.ACTUAL_VALUE); + if (Double.isNaN(actualVal)) actualVal = 0.0; + logger.info("addIndicatorDataToActivity: actual value configKey='{}' parsed actualVal={} (NaN->0)", actualValueConfigKey, actualVal); + + Date origBaseDate = parseDateDefaultToday(row, sheet, config, ImporterConstants.ORIGINAL_BASE_VALUE_DATE); + Date revBaseDate = parseDateDefaultToday(row, sheet, config, ImporterConstants.REVISED_BASE_VALUE_DATE); + Date origTargetDate = parseDateDefaultToday(row, sheet, config, ImporterConstants.ORIGINAL_TARGET_VALUE_DATE); + Date revTargetDate = parseDateDefaultToday(row, sheet, config, ImporterConstants.REVISED_TARGET_VALUE_DATE); + Date actualDate = parseDateDefaultToday(row, sheet, config, ImporterConstants.ACTUAL_VALUE_DATE); + logger.info("addIndicatorDataToActivity: actualDate={}", actualDate); + + double baseOrigVal = hasOrigBase ? origBase : (existingBase != null && existingBase.getOriginalValue() != null ? existingBase.getOriginalValue() : 0.0); + double baseRevVal = hasRevBase ? revBase : (existingBase != null && existingBase.getRevisedValue() != null ? existingBase.getRevisedValue() : 0.0); + double targetOrigVal = Double.isNaN(origTarget) ? 0.0 : origTarget; + double targetRevVal = Double.isNaN(revTarget) ? 0.0 : revTarget; + String unit = getCellValueByConfig(row, sheet, config, ImporterConstants.UNIT_OF_MEASURE); + if (unit != null) unit = unit.trim(); + String actualComment = (unit != null && !unit.isEmpty()) ? "Unit: " + unit : null; + + int merged = 0, created = 0, skipped = 0; + for (String locationName : locationNames) { + logger.info("addIndicatorDataToActivity: processing location '{}' for activityId={} indicator='{}'", locationName, activityId, indicatorName); + AmpActivityLocation activityLocation = null; + if (activity.getLocations() != null) { + for (AmpActivityLocation aal : activity.getLocations()) { + if (aal.getLocation() != null && locationName.equalsIgnoreCase(aal.getLocation().getName())) { + activityLocation = aal; + break; + } + } + } + if (activityLocation == null) { + activityLocation = getOrAddActivityLocationForName(activity, locationName, session); + } + if (activityLocation == null) { + logger.info("addIndicatorDataToActivity: could not resolve or add location '{}' for activityId={}, skipping", locationName, activityId); + skipped++; + continue; + } + logger.info("addIndicatorDataToActivity: activityLocation id={} for '{}'", activityLocation.getLocation() != null ? activityLocation.getLocation().getId() : null, locationName); + + IndicatorActivity ia = findExistingIndicatorActivity(activity, indicator, activityLocation); + if (ia != null) { + logger.info("addIndicatorDataToActivity: merging into existing IndicatorActivity for location '{}' (actual={})", locationName, actualVal); + mergeIndicatorValuesIntoExisting(ia, activityLocation, session, config, + baseOrigVal, origBaseDate, baseRevVal, revBaseDate, + targetOrigVal, origTargetDate, targetRevVal, revTargetDate, + actualVal, actualDate, actualComment); + session.flush(); + merged++; + continue; + } + + logger.info("addIndicatorDataToActivity: creating new IndicatorActivity for activityId={} indicator='{}' location='{}' (actual={})", activityId, indicatorName, locationName, actualVal); + ia = new IndicatorActivity(); + ia.setActivity(activity); + ia.setIndicator(indicator); + ia.setActivityLocation(activityLocation); + + Set values = new HashSet<>(); + if (getKey(config, ImporterConstants.ORIGINAL_BASE_VALUE) != null || getKey(config, ImporterConstants.REVISED_BASE_VALUE) != null || existingBase != null) { + AmpIndicatorValue baseOrig = new AmpIndicatorValue(AmpIndicatorValue.BASE); + baseOrig.setValue(baseOrigVal); + baseOrig.setValueDate(origBaseDate); + baseOrig.setIndicatorConnection(ia); + baseOrig.setActivityLocation(activityLocation); + values.add(baseOrig); + AmpIndicatorValue baseRev = new AmpIndicatorValue(AmpIndicatorValue.REVISED); + baseRev.setValue(baseRevVal); + baseRev.setValueDate(revBaseDate); + baseRev.setIndicatorConnection(ia); + baseRev.setActivityLocation(activityLocation); + values.add(baseRev); + } + if (getKey(config, ImporterConstants.ORIGINAL_TARGET_VALUE) != null || getKey(config, ImporterConstants.REVISED_TARGET_VALUE) != null) { + AmpIndicatorValue tOrig = new AmpIndicatorValue(AmpIndicatorValue.TARGET); + tOrig.setValue(targetOrigVal); + tOrig.setValueDate(origTargetDate); + tOrig.setIndicatorConnection(ia); + tOrig.setActivityLocation(activityLocation); + values.add(tOrig); + AmpIndicatorValue tRev = new AmpIndicatorValue(AmpIndicatorValue.TARGET); + tRev.setValue(targetRevVal); + tRev.setValueDate(revTargetDate); + tRev.setIndicatorConnection(ia); + tRev.setActivityLocation(activityLocation); + values.add(tRev); + } + AmpIndicatorValue actual = new AmpIndicatorValue(AmpIndicatorValue.ACTUAL); + actual.setValue(actualVal); + actual.setValueDate(actualDate); + actual.setIndicatorConnection(ia); + actual.setActivityLocation(activityLocation); // required for OnePager form to show value (filters by activityLocation) + if (actualComment != null) actual.setComment(actualComment); + values.add(actual); + logger.info("addIndicatorDataToActivity: new IndicatorActivity - created ACTUAL value: value={}, valueDate={}, saving child", actualVal, actualDate); + + ia.setValues(values); + if (activity.getIndicators() == null) activity.setIndicators(new HashSet<>()); + activity.getIndicators().add(ia); + session.save(ia); + session.save(actual); + logger.info("addIndicatorDataToActivity: saved IndicatorActivity id={} and ACTUAL AmpIndicatorValue", ia.getId()); + created++; + } + session.flush(); + int indicatorsCountAfter = activity.getIndicators() == null ? 0 : activity.getIndicators().size(); + logger.info("addIndicatorDataToActivity: done for activityId={} indicator='{}' - merged={}, created={}, skipped={}, indicatorsCount before={} after={}", activityId, indicatorName, merged, created, skipped, indicatorsCountBefore, indicatorsCountAfter); + } + + /** + * Finds an existing activity–indicator connection for the same activity, indicator, and location. + * Match is by indicator id and activity location (both null or same location). + */ + private static IndicatorActivity findExistingIndicatorActivity(AmpActivityVersion activity, AmpIndicator indicator, + AmpActivityLocation activityLocation) { + if (activity.getIndicators() == null) return null; + Long indicatorId = indicator != null ? indicator.getIndicatorId() : null; + Long locationId = activityLocation != null && activityLocation.getLocation() != null + ? activityLocation.getLocation().getId() : null; + for (IndicatorActivity ia : activity.getIndicators()) { + if (ia.getIndicator() == null) continue; + if (!Objects.equals(ia.getIndicator().getIndicatorId(), indicatorId)) continue; + Long existingLocId = ia.getActivityLocation() != null && ia.getActivityLocation().getLocation() != null + ? ia.getActivityLocation().getLocation().getId() : null; + if (Objects.equals(existingLocId, locationId)) return ia; + } + return null; + } + + /** + * Merges imported values into an existing indicator connection: updates existing values by type where present, + * adds new values only for types that are missing. Sets activityLocation on values so OnePager form can display them. + */ + private static void mergeIndicatorValuesIntoExisting(IndicatorActivity ia, AmpActivityLocation activityLocation, Session session, Map config, + double baseOrigVal, Date origBaseDate, double baseRevVal, Date revBaseDate, + double targetOrigVal, Date origTargetDate, double targetRevVal, Date revTargetDate, + double actualVal, Date actualDate, String actualComment) { + logger.debug("mergeIndicatorValuesIntoExisting: indicatorConnection id={}, actualVal={}", ia.getId(), actualVal); + Set existing = ia.getValues(); + if (existing == null) { + existing = new HashSet<>(); + ia.setValues(existing); + } + boolean hasBase = getKey(config, ImporterConstants.ORIGINAL_BASE_VALUE) != null || getKey(config, ImporterConstants.REVISED_BASE_VALUE) != null; + boolean hasTarget = getKey(config, ImporterConstants.ORIGINAL_TARGET_VALUE) != null || getKey(config, ImporterConstants.REVISED_TARGET_VALUE) != null; + + // Add a new ACTUAL value only if we don't already have the same value on the same date for this location + if (hasActualWithSameValueAndDate(existing, activityLocation, actualVal, actualDate)) { + logger.info("mergeIndicatorValuesIntoExisting: skipping ACTUAL - same value {} and date {} already present for this location", actualVal, actualDate); + } else { + logger.info("mergeIndicatorValuesIntoExisting: adding new ACTUAL value: value={} valueDate={}", actualVal, actualDate); + AmpIndicatorValue actual = new AmpIndicatorValue(AmpIndicatorValue.ACTUAL); + actual.setValue(actualVal); + actual.setValueDate(actualDate); + if (actualComment != null) actual.setComment(actualComment); + actual.setIndicatorConnection(ia); + actual.setActivityLocation(activityLocation); + existing.add(actual); + session.save(actual); + } + + if (hasBase) { + AmpIndicatorValue existingBase = findValueByType(existing, AmpIndicatorValue.BASE); + if (existingBase != null) { + existingBase.setValue(baseOrigVal); + existingBase.setValueDate(origBaseDate); + if (activityLocation != null && existingBase.getActivityLocation() == null) existingBase.setActivityLocation(activityLocation); + } else { + AmpIndicatorValue baseOrig = new AmpIndicatorValue(AmpIndicatorValue.BASE); + baseOrig.setValue(baseOrigVal); + baseOrig.setValueDate(origBaseDate); + baseOrig.setIndicatorConnection(ia); + baseOrig.setActivityLocation(activityLocation); + existing.add(baseOrig); + session.save(baseOrig); + } + AmpIndicatorValue existingRev = findValueByType(existing, AmpIndicatorValue.REVISED); + if (existingRev != null) { + existingRev.setValue(baseRevVal); + existingRev.setValueDate(revBaseDate); + if (activityLocation != null && existingRev.getActivityLocation() == null) existingRev.setActivityLocation(activityLocation); + } else { + AmpIndicatorValue baseRev = new AmpIndicatorValue(AmpIndicatorValue.REVISED); + baseRev.setValue(baseRevVal); + baseRev.setValueDate(revBaseDate); + baseRev.setIndicatorConnection(ia); + baseRev.setActivityLocation(activityLocation); + existing.add(baseRev); + session.save(baseRev); + } + } + + if (hasTarget) { + List targets = getValuesByType(existing, AmpIndicatorValue.TARGET); + if (targets.size() >= 2) { + targets.get(0).setValue(targetOrigVal); + targets.get(0).setValueDate(origTargetDate); + if (activityLocation != null && targets.get(0).getActivityLocation() == null) targets.get(0).setActivityLocation(activityLocation); + targets.get(1).setValue(targetRevVal); + targets.get(1).setValueDate(revTargetDate); + if (activityLocation != null && targets.get(1).getActivityLocation() == null) targets.get(1).setActivityLocation(activityLocation); + } else if (targets.size() == 1) { + targets.get(0).setValue(targetOrigVal); + targets.get(0).setValueDate(origTargetDate); + if (activityLocation != null && targets.get(0).getActivityLocation() == null) targets.get(0).setActivityLocation(activityLocation); + AmpIndicatorValue tRev = new AmpIndicatorValue(AmpIndicatorValue.TARGET); + tRev.setValue(targetRevVal); + tRev.setValueDate(revTargetDate); + tRev.setIndicatorConnection(ia); + tRev.setActivityLocation(activityLocation); + existing.add(tRev); + session.save(tRev); + } else { + AmpIndicatorValue tOrig = new AmpIndicatorValue(AmpIndicatorValue.TARGET); + tOrig.setValue(targetOrigVal); + tOrig.setValueDate(origTargetDate); + tOrig.setIndicatorConnection(ia); + tOrig.setActivityLocation(activityLocation); + existing.add(tOrig); + session.save(tOrig); + AmpIndicatorValue tRev = new AmpIndicatorValue(AmpIndicatorValue.TARGET); + tRev.setValue(targetRevVal); + tRev.setValueDate(revTargetDate); + tRev.setIndicatorConnection(ia); + tRev.setActivityLocation(activityLocation); + existing.add(tRev); + session.save(tRev); + } + } + evictIndicatorConnectionFromSecondLevelCache(ia); + } + + /** + * Evicts the activity from the second-level cache before an update. Avoids ObjectNotFoundException + * when the cached activity (or its activityContacts) references deleted entities (e.g. AmpActivityDocument). + */ + private static void evictActivityFromSecondLevelCache(Long activityId) { + if (activityId == null) return; + try { + org.hibernate.SessionFactory sessionFactory = org.digijava.kernel.persistence.PersistenceManager.sf(); + if (sessionFactory == null) return; + org.hibernate.Cache cache = sessionFactory.getCache(); + if (cache == null) return; + cache.evictEntityData(AmpActivityVersion.class, activityId); + } catch (Exception e) { + logger.debug("Could not evict activity from cache: {}", e.getMessage()); + } + } + + /** + * Evicts the indicator connection and its values from the second-level cache so the activity form + * sees updated values on next load (otherwise cached stale values can be shown). + */ + private static void evictIndicatorConnectionFromSecondLevelCache(IndicatorActivity ia) { + try { + org.hibernate.SessionFactory sessionFactory = org.digijava.kernel.persistence.PersistenceManager.sf(); + if (sessionFactory == null) return; + org.hibernate.Cache cache = sessionFactory.getCache(); + if (cache == null) return; + if (ia.getId() != null) cache.evictEntityData(IndicatorConnection.class, ia.getId()); + if (ia.getValues() != null) { + for (AmpIndicatorValue v : ia.getValues()) { + if (v != null && v.getIndValId() != null) cache.evictEntityData(AmpIndicatorValue.class, v.getIndValId()); + } + } + } catch (Exception e) { + logger.debug("Could not evict indicator connection from cache: {}", e.getMessage()); + } + } + + private static AmpIndicatorValue findValueByType(Set values, int valueType) { + if (values == null) return null; + for (AmpIndicatorValue v : values) { + if (v.getValueType() == valueType) return v; + } + return null; + } + + /** Returns true if there is already an ACTUAL value for this location with the same value and same date (calendar day). */ + private static boolean hasActualWithSameValueAndDate(Set values, AmpActivityLocation activityLocation, double value, Date valueDate) { + if (values == null) return false; + Long wantLocId = activityLocation != null && activityLocation.getLocation() != null ? activityLocation.getLocation().getId() : null; + for (AmpIndicatorValue v : values) { + if (v.getValueType() != AmpIndicatorValue.ACTUAL) continue; + Long vLocId = v.getActivityLocation() != null && v.getActivityLocation().getLocation() != null + ? v.getActivityLocation().getLocation().getId() : null; + if (!Objects.equals(vLocId, wantLocId)) continue; + if (Double.compare(v.getValue(), value) != 0) continue; + if (!isSameDay(v.getValueDate(), valueDate)) continue; + return true; + } + return false; + } + + /** + * Converts a Date to LocalDate, handling both java.util.Date and java.sql.Date. + * java.sql.Date doesn't support toInstant(), so we use toLocalDate() for it. + */ + private static LocalDate toLocalDate(Date date) { + if (date instanceof java.sql.Date) { + return ((java.sql.Date) date).toLocalDate(); + } + return date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); + } + + private static boolean isSameDay(Date a, Date b) { + if (a == null && b == null) return true; + if (a == null || b == null) return false; + return toLocalDate(a).equals(toLocalDate(b)); + } + + /** Finds ACTUAL value matching this location; falls back to ACTUAL with null location if none match (for backward compat). */ + private static AmpIndicatorValue findValueByTypeAndLocation(Set values, int valueType, AmpActivityLocation activityLocation) { + if (values == null) return null; + AmpIndicatorValue fallbackNull = null; + Long wantLocId = activityLocation != null && activityLocation.getLocation() != null ? activityLocation.getLocation().getId() : null; + for (AmpIndicatorValue v : values) { + if (v.getValueType() != valueType) continue; + Long vLocId = v.getActivityLocation() != null && v.getActivityLocation().getLocation() != null + ? v.getActivityLocation().getLocation().getId() : null; + if (Objects.equals(vLocId, wantLocId)) return v; + if (v.getActivityLocation() == null) fallbackNull = v; + } + return fallbackNull; + } + + private static List getValuesByType(Set values, int valueType) { + if (values == null) return Collections.emptyList(); + List list = new ArrayList<>(); + for (AmpIndicatorValue v : values) { + if (v.getValueType() == valueType) list.add(v); + } + list.sort(Comparator.comparing(AmpIndicatorValue::getValueDate, Comparator.nullsLast(Comparator.naturalOrder()))); + return list; + } + + /** + * Adds the program (theme) to the activity's programs if not already present. + * When the Program Percentage field is enabled, percentages are recalculated and + * divided evenly among all activity programs (including the one just added). + */ + public static void addProgramToActivityIfMissing(AmpActivityVersion activity, AmpTheme program, Session session) { + if (activity == null || program == null) return; + Set actPrograms = activity.getActPrograms(); + if (actPrograms == null) { + actPrograms = new HashSet<>(); + activity.setActPrograms(actPrograms); + } + for (AmpActivityProgram ap : actPrograms) { + if (ap.getProgram() != null && program.getAmpThemeId() != null + && program.getAmpThemeId().equals(ap.getProgram().getAmpThemeId())) { + return; + } + } + AmpActivityProgram activityProgram = new AmpActivityProgram(); + activityProgram.setActivity(activity); + activityProgram.setProgram(program); + activityProgram.setProgramPercentage(100f); + actPrograms.add(activityProgram); + session.save(activityProgram); + + // If Program Percentage field is enabled, distribute 100% evenly among all programs + boolean percentageEnabled = false; + try { + percentageEnabled = FeaturesUtil.isVisibleField(ArConstants.PROGRAM_PERCENTAGE); + } catch (Exception e) { + // No request/session (e.g. batch) – skip percentage redistribution + logger.error("Could not determine if Program Percentage field is enabled; skipping percentage redistribution", e); + } + if (percentageEnabled && !actPrograms.isEmpty()) { + List list = new ArrayList<>(actPrograms); + int n = list.size(); + Map percentages = divide100(n); + for (int i = 0; i < n; i++) { + list.get(i).setProgramPercentage(percentages.get(i)); + } + } + } + + /** + * Returns the program (theme) by name, or creates a new root-level program if it does not exist. + * @param programName program name (must be non-empty) + * @param session current session (used for create and flush) + * @return AmpTheme or null if programName is null/empty or creation fails + */ + public static AmpTheme getOrCreateProgramByName(String programName, Session session) { + if (programName == null || programName.trim().isEmpty()) return null; + programName = programName.trim(); + AmpTheme theme = ProgramUtil.getTheme(programName); + if (theme != null) return theme; + try { + AmpTheme newTheme = new AmpTheme(); + newTheme.setName(programName); + String code = programName.replaceAll("[^a-zA-Z0-9_-]", "_").replaceAll("_+", "_").trim(); + if (code.length() > 45) code = code.substring(0, 45); + newTheme.setThemeCode("IMP_" + code + "_" + System.currentTimeMillis()); + newTheme.setIndlevel(0); + newTheme.setParentThemeId(null); + session.save(newTheme); + session.flush(); + return newTheme; + } catch (Exception e) { + logger.warn("Failed to create program: " + programName, e); + return null; + } + } + + public static String getCellValueByConfig(Row row, Sheet sheet, Map config, String fieldName) { + String key = getKey(config, fieldName); + if (key == null) return null; + int col = getColumnIndexByName(sheet, key); + if (col < 0) return null; + return getStringValueFromCell(row.getCell(col), true); + } + + private static double parseDoubleFromConfig(Row row, Sheet sheet, Map config, String fieldName) { + String key = getKey(config, fieldName); + if (key == null) { + if (ImporterConstants.ACTUAL_VALUE.equals(fieldName)) logger.info("parseDoubleFromConfig: no config key for field '{}'", fieldName); + return Double.NaN; + } + int col = getColumnIndexByName(sheet, key); + if (col < 0) { + if (ImporterConstants.ACTUAL_VALUE.equals(fieldName)) logger.info("parseDoubleFromConfig: column not found for key '{}' in sheet", key); + return Double.NaN; + } + Cell cell = row.getCell(col); + if (cell == null) { + if (ImporterConstants.ACTUAL_VALUE.equals(fieldName)) logger.info("parseDoubleFromConfig: cell is null for col={} key='{}'", col, key); + return Double.NaN; + } + try { + if (cell.getCellType() == Cell.CELL_TYPE_NUMERIC) { + double v = cell.getNumericCellValue(); + if (ImporterConstants.ACTUAL_VALUE.equals(fieldName)) logger.info("parseDoubleFromConfig: ACTUAL_VALUE from numeric cell col='{}' value={}", key, v); + return v; + } + String s = getStringValueFromCell(cell, true); + if (s == null || s.trim().isEmpty()) { + if (ImporterConstants.ACTUAL_VALUE.equals(fieldName)) logger.info("parseDoubleFromConfig: ACTUAL_VALUE cell empty for col='{}'", key); + return Double.NaN; + } + double v = Double.parseDouble(s.trim()); + if (ImporterConstants.ACTUAL_VALUE.equals(fieldName)) logger.info("parseDoubleFromConfig: ACTUAL_VALUE from string cell col='{}' raw='{}' parsed={}", key, s, v); + return v; + } catch (Exception e) { + if (ImporterConstants.ACTUAL_VALUE.equals(fieldName)) logger.info("parseDoubleFromConfig: ACTUAL_VALUE parse failed for col='{}' cellType={} error={}", key, cell.getCellType(), e.getMessage()); + return Double.NaN; + } + } } diff --git a/amp/src/main/java/org/digijava/module/aim/dbentity/IndicatorConnection.java b/amp/src/main/java/org/digijava/module/aim/dbentity/IndicatorConnection.java index 21e575d1449..ff0fa8f8f3c 100644 --- a/amp/src/main/java/org/digijava/module/aim/dbentity/IndicatorConnection.java +++ b/amp/src/main/java/org/digijava/module/aim/dbentity/IndicatorConnection.java @@ -28,11 +28,19 @@ public class IndicatorConnection implements Serializable, Comparable getValues() { @@ -96,5 +105,17 @@ public AmpActivityLocation getActivityLocation() { public void setActivityLocation(AmpActivityLocation activityLocation) { this.activityLocation = activityLocation; + this.indicatorLocationKey = null; // reset so getter recomputes + } + + /** Composite key for uniqueness: indicator id + location id so same indicator with different locations is allowed. */ + public String getIndicatorLocationKey() { + if (indicatorLocationKey == null) { + Long indId = indicator != null ? indicator.getIndicatorId() : null; + Long locId = activityLocation != null && activityLocation.getLocation() != null + ? activityLocation.getLocation().getId() : null; + indicatorLocationKey = (indId != null ? indId : "") + "_" + (locId != null ? locId : ""); + } + return indicatorLocationKey; } } diff --git a/amp/src/main/java/org/digijava/module/aim/util/ActivityUtil.java b/amp/src/main/java/org/digijava/module/aim/util/ActivityUtil.java index 82799e16667..56db388a1c6 100644 --- a/amp/src/main/java/org/digijava/module/aim/util/ActivityUtil.java +++ b/amp/src/main/java/org/digijava/module/aim/util/ActivityUtil.java @@ -450,6 +450,12 @@ public static AmpActivityVersion loadActivity(Long id) throws DgException { Hibernate.initialize(str.getType()); Hibernate.initialize(str.getCoordinates()); } + Hibernate.initialize(result.getIndicators()); + if (result.getIndicators() != null) { + for (IndicatorActivity ia : result.getIndicators()) { + Hibernate.initialize(ia.getValues()); + } + } // AMPOFFLINE-1528 ActivityUtil.setCurrentWorkspacePrefixIntoRequest(result); diff --git a/amp/src/main/java/org/digijava/module/message/jobs/AmpDonorFundingJob.java b/amp/src/main/java/org/digijava/module/message/jobs/AmpDonorFundingJob.java index 3e28ec4261f..84c625badb7 100644 --- a/amp/src/main/java/org/digijava/module/message/jobs/AmpDonorFundingJob.java +++ b/amp/src/main/java/org/digijava/module/message/jobs/AmpDonorFundingJob.java @@ -485,7 +485,7 @@ public static void sendReportsToServer(List ampDashboardFundin // Convert to JSON using a JSON library (e.g., Gson) Gson gson = new Gson(); String jsonData = gson.toJson(submissionData); - logger.info("JSON data: " + jsonData); +// logger.info("JSON data: " + jsonData); // Get the output stream of the connection try (OutputStream os = connection.getOutputStream()) { diff --git a/amp/src/main/resources/xmlpatches/4.0/AMP-30885-Data-Importer-Menu-Items-v3.xml b/amp/src/main/resources/xmlpatches/4.0/AMP-30885-Data-Importer-Menu-Items-v3.xml deleted file mode 100644 index fca110892ba..00000000000 --- a/amp/src/main/resources/xmlpatches/4.0/AMP-30885-Data-Importer-Menu-Items-v3.xml +++ /dev/null @@ -1,56 +0,0 @@ - - - AMP-30885 - menu - bmokandu - Add menu entry for Data Importer - - - - diff --git a/amp/src/main/resources/xmlpatches/4.0/AMP-30885-Data-Importer-Menu-Items-v4.xml b/amp/src/main/resources/xmlpatches/4.0/AMP-30885-Data-Importer-Menu-Items-v4.xml new file mode 100644 index 00000000000..91a3ed4bea8 --- /dev/null +++ b/amp/src/main/resources/xmlpatches/4.0/AMP-30885-Data-Importer-Menu-Items-v4.xml @@ -0,0 +1,67 @@ + + + AMP-30885 + menu + bmokandu + Add menu entry for Data Importer + + + + diff --git a/amp/src/main/resources/xmlpatches/4.0/Increase-ImportedProjectCurrency-Name-Length.xml b/amp/src/main/resources/xmlpatches/4.0/Increase-ImportedProjectCurrency-Name-Length.xml new file mode 100644 index 00000000000..7e8e36233d7 --- /dev/null +++ b/amp/src/main/resources/xmlpatches/4.0/Increase-ImportedProjectCurrency-Name-Length.xml @@ -0,0 +1,20 @@ + + + AMP-DATAIMPORT + Increase imported_project_name column length to support long project names + brianbrix + + Increases the IMPORTED_PROJECT_CURRENCY.imported_project_name column from varchar(255) to varchar(2000) + to accommodate lengthy project names that may exceed 255 characters. + + + + + diff --git a/amp/src/main/webapp/WEB-INF/jsp/aim/view/dataImporter.jsp b/amp/src/main/webapp/WEB-INF/jsp/aim/view/dataImporter.jsp index c24461951be..6aa5ad4350e 100644 --- a/amp/src/main/webapp/WEB-INF/jsp/aim/view/dataImporter.jsp +++ b/amp/src/main/webapp/WEB-INF/jsp/aim/view/dataImporter.jsp @@ -20,15 +20,24 @@ } } function addField() { - var columnName = document.getElementById("columnName").value; + var columnName = ($('#current-config-name').val()) + ? $('#column-name-edit').val() + : document.getElementById("columnName").value; var selectedField = document.getElementById("selected-field").value; - - sendValuesToBackend(columnName,selectedField,"addField") - - + if (!columnName || !selectedField) { + alert("Please enter column name and select a field."); + return; + } + sendValuesToBackend(columnName, selectedField, "addField"); + if ($('#current-config-name').val()) { + $('#column-name-edit').val(''); + } } $(document).ready(function() { $('#existing-config').val('0'); + if ($('#file-type').val() === 'excel') { + $('#data-sheet-choice-div').show(); + } $('.remove-row').click(function() { var selectedRows = $('.fields-table tbody').find('.remove-checkbox:checked').closest('tr'); @@ -54,30 +63,71 @@ $('#data-file').attr("accept", ".csv"); $('#template-file').attr("accept", ".csv"); $('#separator-div').hide(); - + $('#data-sheet-choice-div').hide(); } else if(fileType==="text") { $('#select-file-label').html("Select text file"); $('#data-file').attr("accept", ".txt"); $('#template-file').attr("accept", ".txt"); $('#separator-div').show(); + $('#data-sheet-choice-div').hide(); } else if(fileType==="excel") { $('#select-file-label').html("Select excel file"); $('#data-file').attr("accept", ".xls,.xlsx"); $('#template-file').attr("accept", ".xls,.xlsx"); $('#separator-div').hide(); + $('#data-sheet-choice-div').show(); } else if(fileType==="json") { $('#select-file-label').html("Select json file"); $('#data-file').attr("accept", ".json"); $('#template-file').attr("accept", ".json"); $('#separator-div').hide(); + $('#data-sheet-choice-div').hide(); + } + }); + + $('input[name="dataSheetChoice"]').change(function() { + var v = $(this).val(); + if (v === 'sheet') { + $('#data-sheet-select-wrap').show(); + } else { + $('#data-sheet-select-wrap').hide(); } }); + $('#load-sheets-btn').click(function() { + var fileInput = document.getElementById('data-file'); + if (!fileInput.files || !fileInput.files.length) { + alert("Please select a data file first."); + return; + } + var formData = new FormData(); + formData.append('dataFile', fileInput.files[0]); + formData.append('action', 'getDataFileSheets'); + formData.append('fileType', $('#file-type').val()); + var $select = $('#data-sheet'); + $select.prop('disabled', true).empty().append(''); + fetch("${pageContext.request.contextPath}/aim/dataImporter.do", { method: "POST", body: formData }) + .then(function(r) { return r.json(); }) + .then(function(names) { + $select.empty().append(''); + if (Array.isArray(names)) { + names.forEach(function(name) { + $select.append($('').attr('value', name).text(name)); + }); + $select.prop('disabled', false); + } + }) + .catch(function() { + $select.empty().append('').prop('disabled', false); + alert("Could not load sheets from file."); + }); + }); + $('.existing-config').change(function() { var configName = $(this).val(); - if (configName!=='none'){ + if (configName!=='none' && configName!=='0'){ var formData = new FormData(); formData.append("configName", configName); @@ -94,33 +144,28 @@ console.log("Response: ",response); $("#templateUploadForm").hide(); $('#existing-config').val('0'); + $('#current-config-name').val(configName); return response.json(); }) .then(updatedMap => { console.log("Map :" ,updatedMap) - // Update UI or perform any additional actions if needed - console.log("Selected pairs updated successfully."); - console.log("Updated map received:", updatedMap); var tbody = document.getElementById("selected-pairs-table-body"); - - // Remove all rows from the table body tbody.innerHTML = ""; for (var key in updatedMap) { if (updatedMap.hasOwnProperty(key)) { - // Access each property using the key var value = updatedMap[key]; updateTable(key, value, tbody); console.log('Key:', key, 'Value:', value); } } document.getElementById("otherComponents").removeAttribute("hidden"); - $('#add-field').hide(); - $('.remove-row').hide(); - $('#selected-field').hide(); - $('#existing-config').val('1'); - + $('#add-field').show(); + $('.remove-row').show(); + $('#selected-field').show(); + $('#add-pair-edit-section').show(); + $('#column-name-edit').val(''); }) .catch(error => { @@ -128,6 +173,8 @@ }); }else { + $('#current-config-name').val(''); + $('#add-pair-edit-section').hide(); $("#templateUploadForm").show(); } }); @@ -135,11 +182,14 @@ function sendValuesToBackend(columnName, selectedField, action) { - // Create a FormData object to send data in the request body var formData = new FormData(); formData.append("columnName", columnName); formData.append("selectedField", selectedField); formData.append("action", action); + var configName = $('#current-config-name').val(); + if (configName) { + formData.append("configName", configName); + } fetch("${pageContext.request.contextPath}/aim/dataImporter.do", { method: "POST", @@ -225,13 +275,24 @@ var xhr = new XMLHttpRequest(); xhr.open('POST', '${pageContext.request.contextPath}/aim/dataImporter.do', true); - xhr.setRequestHeader("Accept", "text/html"); // Ensure response is HTML + xhr.setRequestHeader("Accept", "application/json, text/html"); xhr.onload = function () { if (xhr.status === 200) { if (xhr.responseText && xhr.responseText.trim().length >= 1) { - // Set response HTML inside the headers div - document.getElementById('headers').innerHTML = xhr.responseText; + var ct = xhr.getResponseHeader("Content-Type") || ""; + if (ct.indexOf("application/json") !== -1) { + try { + var data = JSON.parse(xhr.responseText); + renderTemplateSheetAndColumns(data); + } catch (e) { + console.error("Invalid JSON response", e); + alert("Unable to parse template. Please try again."); + return; + } + } else { + document.getElementById('headers').innerHTML = xhr.responseText; + } alert("The template has been successfully uploaded."); document.getElementById("otherComponents").removeAttribute("hidden"); $('#add-field').show(); @@ -251,13 +312,56 @@ xhr.send(formData); } + function renderTemplateSheetAndColumns(data) { + var sheetNames = data.sheetNames || []; + var columnsBySheet = data.columnsBySheet || {}; + var headersDiv = document.getElementById('headers'); + if (sheetNames.length === 0) { + headersDiv.innerHTML = '

No sheets found in the template.

'; + return; + } + var firstSheet = sheetNames[0]; + var html = '
'; + html += '

'; + html += '
'; + html += ''; + headersDiv.innerHTML = html; + + window._templateColumnsBySheet = columnsBySheet; + $('#template-sheet').off('change.templateColumns').on('change.templateColumns', function() { + var sheet = $(this).val(); + var cols = window._templateColumnsBySheet[sheet] || []; + var $colSelect = $('#columnName'); + $colSelect.empty(); + for (var k = 0; k < cols.length; k++) { + $colSelect.append($('').text(cols[k])); + } + }); + } + + function escapeHtml(text) { + var div = document.createElement('div'); + div.textContent = text; + return div.innerHTML; + } + function uploadDataFile() { var formData = new FormData(); var fileType = $('#file-type').val(); var internal = $('#internal').prop('checked'); console.log("Internal", internal); var dataSeparator = $('#data-separator').val(); - var existingConfig = $('#existing-config').val(); + var currentConfigName = $('#current-config-name').val(); + var existingConfig = (currentConfigName && currentConfigName.trim() !== '') ? currentConfigName.trim() : $('#existing-config').val(); console.log("Existing configuration: " + existingConfig); var fileInput = document.getElementById('data-file'); // Check if a file is selected @@ -265,12 +369,23 @@ alert("Please select a file to upload."); return; } + var dataSheetChoice = $('input[name="dataSheetChoice"]:checked').val(); + var dataSheetName = $('#data-sheet').val() || ''; + if (fileType === 'excel' && dataSheetChoice === 'sheet' && !dataSheetName) { + alert("Please load sheets and select a sheet, or choose 'Whole file'."); + return; + } formData.append('dataFile', fileInput.files[0]); formData.append('internal', internal); formData.append('action',"uploadDataFile"); formData.append('fileType', fileType); formData.append('dataSeparator', dataSeparator); formData.append('existingConfig', existingConfig); + if (currentConfigName && currentConfigName.trim() !== '') { + formData.append('configName', currentConfigName.trim()); + } + formData.append('dataSheetChoice', dataSheetChoice || 'all'); + formData.append('dataSheetName', dataSheetName); var xhr = new XMLHttpRequest(); xhr.open('POST', '${pageContext.request.contextPath}/aim/dataImporter.do', true); @@ -369,11 +484,16 @@