From e03662c97704147a56bfda0dd6990fe96dbdae14 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Fri, 13 Feb 2026 11:42:08 +0300 Subject: [PATCH 01/59] Adapt data importer for GGW --- .../aim/action/dataimporter/DataImporter.java | 197 ++++++++++++------ .../action/dataimporter/ExcelImporter.java | 72 +++++-- .../action/dataimporter/TxtDataImporter.java | 2 + .../WEB-INF/jsp/aim/view/dataImporter.jsp | 126 ++++++++++- 4 files changed, 311 insertions(+), 86 deletions(-) diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java index cc5244d9b79..4aa69703e6d 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java @@ -1,11 +1,34 @@ package org.digijava.module.aim.action.dataimporter; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.opencsv.CSVParser; -import com.opencsv.CSVParserBuilder; -import com.opencsv.CSVReader; -import com.opencsv.CSVReaderBuilder; -import com.opencsv.exceptions.CsvValidationException; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.file.Files; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; @@ -16,11 +39,16 @@ import org.apache.struts.action.ActionForward; import org.apache.struts.action.ActionMapping; import org.digijava.kernel.persistence.PersistenceManager; +import static org.digijava.module.aim.action.dataimporter.ExcelImporter.processExcelFileInBatches; import org.digijava.module.aim.action.dataimporter.dbentity.DataImporterConfig; import org.digijava.module.aim.action.dataimporter.dbentity.DataImporterConfigValues; import org.digijava.module.aim.action.dataimporter.dbentity.ImportStatus; import org.digijava.module.aim.action.dataimporter.dbentity.ImportedFilesRecord; import org.digijava.module.aim.action.dataimporter.util.ImportedFileUtil; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.ConstantsMap; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.isFileContentValid; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.isFileReadable; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.removeMapItem; import org.digijava.module.aim.form.DataImporterForm; import org.hibernate.Query; import org.hibernate.Session; @@ -28,22 +56,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.*; -import java.nio.file.Files; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.time.Duration; -import java.time.Instant; -import java.time.LocalDateTime; -import java.util.*; -import java.util.stream.Collectors; - -import static org.digijava.module.aim.action.dataimporter.ExcelImporter.processExcelFileInBatches; -import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.*; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.opencsv.CSVParser; +import com.opencsv.CSVParserBuilder; +import com.opencsv.CSVReader; +import com.opencsv.CSVReaderBuilder; +import com.opencsv.exceptions.CsvValidationException; public class DataImporter extends Action { static Logger logger = LoggerFactory.getLogger(DataImporter.class); @@ -81,59 +99,87 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet logger.info(" this is the action " + request.getParameter("action")); if (request.getParameter("uploadTemplate") != null) { logger.info(" this is the action " + request.getParameter("uploadTemplate")); - Set headersSet = new HashSet<>(); + response.setCharacterEncoding("UTF-8"); + dataImporterForm.getColumnPairs().clear(); if (request.getParameter("fileType") != null) { InputStream fileInputStream = dataImporterForm.getTemplateFile().getInputStream(); - if ((Objects.equals(request.getParameter("fileType"), "excel") || Objects.equals(request.getParameter("fileType"), "csv"))) { - Workbook workbook = new XSSFWorkbook(fileInputStream); - int numberOfSheets = workbook.getNumberOfSheets(); - for (int i = 0; i < numberOfSheets; i++) { - Sheet sheet = workbook.getSheetAt(i); - Row headerRow = sheet.getRow(0); - Iterator cellIterator = headerRow.cellIterator(); - while (cellIterator.hasNext()) { - Cell cell = cellIterator.next(); - headersSet.add(cell.getStringCellValue()); + if (Objects.equals(request.getParameter("fileType"), "excel")) { + // Excel: return sheet names and columns per sheet for template configuration + List sheetNames = new ArrayList<>(); + Map> columnsBySheet = new HashMap<>(); + try (Workbook workbook = new XSSFWorkbook(fileInputStream)) { + int numberOfSheets = workbook.getNumberOfSheets(); + for (int i = 0; i < numberOfSheets; i++) { + Sheet sheet = workbook.getSheetAt(i); + String sheetName = sheet.getSheetName(); + sheetNames.add(sheetName); + List columns = new ArrayList<>(); + Row headerRow = sheet.getRow(0); + if (headerRow != null) { + Iterator cellIterator = headerRow.cellIterator(); + while (cellIterator.hasNext()) { + Cell cell = cellIterator.next(); + String val = cell.getStringCellValue(); + if (val != null && !val.trim().isEmpty()) { + columns.add(val.trim()); + } + } + } + columnsBySheet.put(sheetName, columns.stream().sorted().collect(Collectors.toList())); } - } - workbook.close(); - - + Map jsonResponse = new HashMap<>(); + jsonResponse.put("sheetNames", sheetNames); + jsonResponse.put("columnsBySheet", columnsBySheet); + response.setContentType("application/json"); + new ObjectMapper().writeValue(response.getWriter(), jsonResponse); + } else if (Objects.equals(request.getParameter("fileType"), "csv")) { + Set headersSet = new HashSet<>(); + try (CSVReader reader = new CSVReaderBuilder(new InputStreamReader(fileInputStream)).build()) { + String[] headers = reader.readNext(); + if (headers != null) { + headersSet.addAll(Arrays.asList(headers)); + } + } catch (IOException | CsvValidationException e) { + logger.error("An error occurred during extraction of headers.", e); + } + headersSet = headersSet.stream().sorted().collect(Collectors.toCollection(LinkedHashSet::new)); + StringBuilder headers = new StringBuilder(); + headers.append(" \n"); + response.setContentType("text/html;charset=UTF-8"); + response.getWriter().write(headers.toString()); } else if (Objects.equals(request.getParameter("fileType"), "text")) { - CSVParser parser = new CSVParserBuilder().withSeparator(request.getParameter("dataSeparator").charAt(0)).build(); - + Set headersSet = new HashSet<>(); + String sep = request.getParameter("dataSeparator"); + char separator = (sep != null && !sep.isEmpty()) ? sep.charAt(0) : ','; + CSVParser parser = new CSVParserBuilder().withSeparator(separator).build(); try (CSVReader reader = new CSVReaderBuilder(new InputStreamReader(fileInputStream)).withCSVParser(parser).build()) { - String[] headers = reader.readNext(); // Read the first line which contains headers - + String[] headers = reader.readNext(); if (headers != null) { - // Print each header headersSet.addAll(Arrays.asList(headers)); } else { logger.info("File is empty or does not contain headers."); } } catch (IOException | CsvValidationException e) { - logger.error("An error occurred during extraction of headers.",e); + logger.error("An error occurred during extraction of headers.", e); } - + headersSet = headersSet.stream().sorted().collect(Collectors.toCollection(LinkedHashSet::new)); + StringBuilder headers = new StringBuilder(); + headers.append(" \n"); + response.setContentType("text/html;charset=UTF-8"); + response.getWriter().write(headers.toString()); } } - headersSet = headersSet.stream().sorted().collect(Collectors.toCollection(LinkedHashSet::new)); - StringBuilder headers = new StringBuilder(); - headers.append(" \n"); - response.setCharacterEncoding("UTF-8"); - response.setContentType("text/html;charset=UTF-8"); - - response.getWriter().write(headers.toString()); response.setHeader("updatedMap", ""); - - dataImporterForm.getColumnPairs().clear(); - } return null; } @@ -181,6 +227,34 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet } + if (Objects.equals(request.getParameter("action"), "getDataFileSheets")) { + logger.info("This is the action getDataFileSheets"); + if (dataImporterForm.getDataFile() == null || dataImporterForm.getDataFile().getFileSize() == 0) { + response.setStatus(400); + response.setContentType("application/json"); + response.getWriter().write("{\"error\":\"No file provided\"}"); + return null; + } + String fileType = request.getParameter("fileType"); + if (!Objects.equals(fileType, "excel")) { + response.setContentType("application/json"); + new ObjectMapper().writeValue(response.getWriter(), Collections.emptyList()); + return null; + } + List sheetNames = new ArrayList<>(); + try (InputStream is = dataImporterForm.getDataFile().getInputStream(); + Workbook workbook = new XSSFWorkbook(is)) { + int n = workbook.getNumberOfSheets(); + for (int i = 0; i < n; i++) { + sheetNames.add(workbook.getSheetAt(i).getSheetName()); + } + } + response.setContentType("application/json"); + response.setCharacterEncoding("UTF-8"); + new ObjectMapper().writeValue(response.getWriter(), sheetNames); + return null; + } + if (Objects.equals(request.getParameter("action"), "uploadDataFile")) { logger.info("This is the action " + request.getParameter("action")); Instant start = Instant.now(); @@ -245,9 +319,11 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet } logger.info("Configuration"+ dataImporterForm.getColumnPairs()); if ((Objects.equals(request.getParameter("fileType"), "excel") || Objects.equals(request.getParameter("fileType"), "csv"))) { - + String dataSheetChoice = request.getParameter("dataSheetChoice"); + String dataSheetName = request.getParameter("dataSheetName"); + boolean useSpecificSheet = "sheet".equals(dataSheetChoice) && dataSheetName != null && !dataSheetName.trim().isEmpty(); // Process the file in batches - res = processExcelFileInBatches(importedFilesRecord, tempFile, request, dataImporterForm.getColumnPairs(), isInternal); + res = processExcelFileInBatches(importedFilesRecord, tempFile, request, dataImporterForm.getColumnPairs(), isInternal, useSpecificSheet ? dataSheetName : null); } else if ( Objects.equals(request.getParameter("fileType"), "text")) { res=TxtDataImporter.processTxtFileInBatches(importedFilesRecord, tempFile, request, dataImporterForm.getColumnPairs(), isInternal); } @@ -388,6 +464,7 @@ private List getEntityFieldsInfo() { List fieldsInfos = new ArrayList<>(); fieldsInfos.add("Project Title"); fieldsInfos.add("Project Code"); + fieldsInfos.add("Objective"); fieldsInfos.add("Project Description"); fieldsInfos.add("Primary Sector"); fieldsInfos.add("Secondary Sector"); diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java index d41c5e64c98..1a9514fd660 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java @@ -1,6 +1,17 @@ package org.digijava.module.aim.action.dataimporter; -import com.fasterxml.jackson.core.JsonProcessingException; +import java.io.File; +import java.io.IOException; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +import javax.servlet.http.HttpServletRequest; + import org.apache.poi.openxml4j.exceptions.InvalidFormatException; import org.apache.poi.openxml4j.exceptions.InvalidOperationException; import org.apache.poi.ss.usermodel.Cell; @@ -16,6 +27,16 @@ import org.digijava.module.aim.action.dataimporter.model.Funding; import org.digijava.module.aim.action.dataimporter.model.ImportDataModel; import org.digijava.module.aim.action.dataimporter.util.ImportedFileUtil; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.existingActivity; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.getColumnIndexByName; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.getKey; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.getStringValueFromCell; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.importTheData; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.setAFundingItemForExcel; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.setStatus; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.updateLocations; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.updateOrgs; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.updateSectors; import org.digijava.module.aim.dbentity.AmpActivityVersion; import org.digijava.module.aim.util.FeaturesUtil; import org.digijava.module.aim.util.TeamMemberUtil; @@ -23,44 +44,48 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.servlet.http.HttpServletRequest; -import java.io.File; -import java.io.IOException; -import java.time.OffsetDateTime; -import java.time.ZoneOffset; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.*; +import com.fasterxml.jackson.core.JsonProcessingException; public class ExcelImporter { static Logger logger = LoggerFactory.getLogger(ExcelImporter.class); private static final int BATCH_SIZE = 1000; public static int processExcelFileInBatches(ImportedFilesRecord importedFilesRecord, File file, HttpServletRequest request, Map config, boolean isInternal) { - int res=0; + return processExcelFileInBatches(importedFilesRecord, file, request, config, isInternal, null); + } + + public static int processExcelFileInBatches(ImportedFilesRecord importedFilesRecord, File file, HttpServletRequest request, Map config, boolean isInternal, String sheetNameToProcess) { + int res = 0; ImportedFileUtil.updateFileStatus(importedFilesRecord, ImportStatus.IN_PROGRESS); try (Workbook workbook = new XSSFWorkbook(file)) { int numberOfSheets = workbook.getNumberOfSheets(); logger.info("Number of sheets: {}", numberOfSheets); - // Process each sheet in the workbook - for (int i = 0; i < numberOfSheets; i++) { - logger.info("Sheet number: {}", i); - Sheet sheet = workbook.getSheetAt(i); + if (sheetNameToProcess != null && !sheetNameToProcess.trim().isEmpty()) { + Sheet sheet = workbook.getSheet(sheetNameToProcess); + if (sheet == null) { + logger.error("Sheet not found: {}", sheetNameToProcess); + ImportedFileUtil.updateFileStatus(importedFilesRecord, ImportStatus.FAILED); + return 0; + } if (isInternal) { addDonorAgencyColumn(sheet, FeaturesUtil.getGlobalSettingValue("Internal Ecowas Donor")); - } - - processSheetInBatches(sheet, request,config, importedFilesRecord); + processSheetInBatches(sheet, request, config, importedFilesRecord); + } else { + // Process each sheet in the workbook + for (int i = 0; i < numberOfSheets; i++) { + logger.info("Sheet number: {}", i); + Sheet sheet = workbook.getSheetAt(i); + if (isInternal) { + addDonorAgencyColumn(sheet, FeaturesUtil.getGlobalSettingValue("Internal Ecowas Donor")); + } + processSheetInBatches(sheet, request, config, importedFilesRecord); + } } logger.info("Closing the workbook..."); - res =1; + res = 1; } catch (IOException e) { ImportedFileUtil.updateFileStatus(importedFilesRecord, ImportStatus.FAILED); logger.error("Error processing Excel file: {}", e.getMessage(), e); @@ -171,6 +196,9 @@ public static void processBatch(List batch,Sheet sheet, HttpServletRequest int projectTitleColumn = getColumnIndexByName(sheet, getKey(config, "Project Title")); String projectTitle = projectTitleColumn >= 0 ? getStringValueFromCell(row.getCell(projectTitleColumn),false) : ""; importDataModel.setProject_title(projectTitle); + int objectiveColumn = getColumnIndexByName(sheet, getKey(config, "Objective")); + String objective = objectiveColumn >= 0 ? getStringValueFromCell(row.getCell(objectiveColumn),false) : null; + importDataModel.setObjective(objective); int projectDescColumn = getColumnIndexByName(sheet, getKey(config, "Project Description")); String projectDesc = projectDescColumn >= 0 ? getStringValueFromCell(row.getCell(projectDescColumn),false) : null; diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java index 1c735c6f9bb..7a3dfcd8c08 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java @@ -102,6 +102,7 @@ private static void processBatch(List> batch, HttpServletRe String projectCode= row.get(getKey(config, "Project Code")); String projectTitle= row.get(getKey(config, "Project Title")); String projectDesc= row.get(getKey(config, "Project Description")); + String objective= row.get(getKey(config, "Objective")); String primarySubSector= row.get(getKey(config, "Primary Subsector")); String secondarySubSector= row.get(getKey(config, "Secondary Subsector")); AmpActivityVersion existing = existingActivity(projectTitle,projectCode,session); @@ -113,6 +114,7 @@ private static void processBatch(List> batch, HttpServletRe } importDataModel.setProject_title(projectTitle); + importDataModel.setObjective(objective); importDataModel.setProject_code(projectCode); importDataModel.setDescription(projectDesc); diff --git a/amp/src/main/webapp/WEB-INF/jsp/aim/view/dataImporter.jsp b/amp/src/main/webapp/WEB-INF/jsp/aim/view/dataImporter.jsp index c24461951be..ae5ab8a59d0 100644 --- a/amp/src/main/webapp/WEB-INF/jsp/aim/view/dataImporter.jsp +++ b/amp/src/main/webapp/WEB-INF/jsp/aim/view/dataImporter.jsp @@ -29,6 +29,9 @@ } $(document).ready(function() { $('#existing-config').val('0'); + if ($('#file-type').val() === 'excel') { + $('#data-sheet-choice-div').show(); + } $('.remove-row').click(function() { var selectedRows = $('.fields-table tbody').find('.remove-checkbox:checked').closest('tr'); @@ -54,27 +57,68 @@ $('#data-file').attr("accept", ".csv"); $('#template-file').attr("accept", ".csv"); $('#separator-div').hide(); - + $('#data-sheet-choice-div').hide(); } else if(fileType==="text") { $('#select-file-label').html("Select text file"); $('#data-file').attr("accept", ".txt"); $('#template-file').attr("accept", ".txt"); $('#separator-div').show(); + $('#data-sheet-choice-div').hide(); } else if(fileType==="excel") { $('#select-file-label').html("Select excel file"); $('#data-file').attr("accept", ".xls,.xlsx"); $('#template-file').attr("accept", ".xls,.xlsx"); $('#separator-div').hide(); + $('#data-sheet-choice-div').show(); } else if(fileType==="json") { $('#select-file-label').html("Select json file"); $('#data-file').attr("accept", ".json"); $('#template-file').attr("accept", ".json"); $('#separator-div').hide(); + $('#data-sheet-choice-div').hide(); } }); + $('input[name="dataSheetChoice"]').change(function() { + var v = $(this).val(); + if (v === 'sheet') { + $('#data-sheet-select-wrap').show(); + } else { + $('#data-sheet-select-wrap').hide(); + } + }); + + $('#load-sheets-btn').click(function() { + var fileInput = document.getElementById('data-file'); + if (!fileInput.files || !fileInput.files.length) { + alert("Please select a data file first."); + return; + } + var formData = new FormData(); + formData.append('dataFile', fileInput.files[0]); + formData.append('action', 'getDataFileSheets'); + formData.append('fileType', $('#file-type').val()); + var $select = $('#data-sheet'); + $select.prop('disabled', true).empty().append(''); + fetch("${pageContext.request.contextPath}/aim/dataImporter.do", { method: "POST", body: formData }) + .then(function(r) { return r.json(); }) + .then(function(names) { + $select.empty().append(''); + if (Array.isArray(names)) { + names.forEach(function(name) { + $select.append($('').attr('value', name).text(name)); + }); + $select.prop('disabled', false); + } + }) + .catch(function() { + $select.empty().append('').prop('disabled', false); + alert("Could not load sheets from file."); + }); + }); + $('.existing-config').change(function() { var configName = $(this).val(); if (configName!=='none'){ @@ -225,13 +269,24 @@ var xhr = new XMLHttpRequest(); xhr.open('POST', '${pageContext.request.contextPath}/aim/dataImporter.do', true); - xhr.setRequestHeader("Accept", "text/html"); // Ensure response is HTML + xhr.setRequestHeader("Accept", "application/json, text/html"); xhr.onload = function () { if (xhr.status === 200) { if (xhr.responseText && xhr.responseText.trim().length >= 1) { - // Set response HTML inside the headers div - document.getElementById('headers').innerHTML = xhr.responseText; + var ct = xhr.getResponseHeader("Content-Type") || ""; + if (ct.indexOf("application/json") !== -1) { + try { + var data = JSON.parse(xhr.responseText); + renderTemplateSheetAndColumns(data); + } catch (e) { + console.error("Invalid JSON response", e); + alert("Unable to parse template. Please try again."); + return; + } + } else { + document.getElementById('headers').innerHTML = xhr.responseText; + } alert("The template has been successfully uploaded."); document.getElementById("otherComponents").removeAttribute("hidden"); $('#add-field').show(); @@ -251,6 +306,48 @@ xhr.send(formData); } + function renderTemplateSheetAndColumns(data) { + var sheetNames = data.sheetNames || []; + var columnsBySheet = data.columnsBySheet || {}; + var headersDiv = document.getElementById('headers'); + if (sheetNames.length === 0) { + headersDiv.innerHTML = '

No sheets found in the template.

'; + return; + } + var firstSheet = sheetNames[0]; + var html = '
'; + html += '

'; + html += '
'; + html += ''; + headersDiv.innerHTML = html; + + window._templateColumnsBySheet = columnsBySheet; + $('#template-sheet').off('change.templateColumns').on('change.templateColumns', function() { + var sheet = $(this).val(); + var cols = window._templateColumnsBySheet[sheet] || []; + var $colSelect = $('#columnName'); + $colSelect.empty(); + for (var k = 0; k < cols.length; k++) { + $colSelect.append($('').text(cols[k])); + } + }); + } + + function escapeHtml(text) { + var div = document.createElement('div'); + div.textContent = text; + return div.innerHTML; + } + function uploadDataFile() { var formData = new FormData(); var fileType = $('#file-type').val(); @@ -265,12 +362,20 @@ alert("Please select a file to upload."); return; } + var dataSheetChoice = $('input[name="dataSheetChoice"]:checked').val(); + var dataSheetName = $('#data-sheet').val() || ''; + if (fileType === 'excel' && dataSheetChoice === 'sheet' && !dataSheetName) { + alert("Please load sheets and select a sheet, or choose 'Whole file'."); + return; + } formData.append('dataFile', fileInput.files[0]); formData.append('internal', internal); formData.append('action',"uploadDataFile"); formData.append('fileType', fileType); formData.append('dataSeparator', dataSeparator); formData.append('existingConfig', existingConfig); + formData.append('dataSheetChoice', dataSheetChoice || 'all'); + formData.append('dataSheetName', dataSheetName); var xhr = new XMLHttpRequest(); xhr.open('POST', '${pageContext.request.contextPath}/aim/dataImporter.do', true); @@ -412,6 +517,19 @@ <%-- --%> + +

: From e482b9bf9507ba2c61eebdf114990d5bbc0b4b51 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Fri, 13 Feb 2026 12:15:51 +0300 Subject: [PATCH 02/59] Add xml patch for Menu item Add method to get indicator by name and program --- .../manager/IndicatorManagerService.java | 27 ++++++++ .../message/jobs/AmpDonorFundingJob.java | 2 +- .../AMP-30885-Data-Importer-Menu-Items-v3.xml | 56 ---------------- .../AMP-30885-Data-Importer-Menu-Items-v4.xml | 67 +++++++++++++++++++ 4 files changed, 95 insertions(+), 57 deletions(-) delete mode 100644 amp/src/main/resources/xmlpatches/4.0/AMP-30885-Data-Importer-Menu-Items-v3.xml create mode 100644 amp/src/main/resources/xmlpatches/4.0/AMP-30885-Data-Importer-Menu-Items-v4.xml diff --git a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/indicator/manager/IndicatorManagerService.java b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/indicator/manager/IndicatorManagerService.java index fa6b9c2e4bc..c043e7e09f1 100644 --- a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/indicator/manager/IndicatorManagerService.java +++ b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/indicator/manager/IndicatorManagerService.java @@ -79,6 +79,33 @@ public MEIndicatorDTO getMEIndicatorById(final Long indicatorId) { throw new ApiRuntimeException(BAD_REQUEST, ApiError.toError("Indicator with id " + indicatorId + " not found")); } + public MEIndicatorDTO getMeIndicatorByNameAndProgramName(String name, String programName) { + Session session = PersistenceManager.getSession(); + AmpIndicator indicator; + if (programName==null){ + indicator = (AmpIndicator) session.createCriteria(AmpIndicator.class) + .add(Restrictions.eq("name", name)) + .setMaxResults(1) + .uniqueResult(); + } + else { + indicator = (AmpIndicator) session.createCriteria(AmpIndicator.class) + .add(Restrictions.eq("name", name)) + .createAlias("program", "p") + .add(Restrictions.eq("p.name", programName)) + .setMaxResults(1) + .uniqueResult(); + } + + + + if (indicator != null) { + return new MEIndicatorDTO(indicator); + } + + throw new ApiRuntimeException(BAD_REQUEST, + ApiError.toError("Indicator with name " + name + " and program name " + programName + " not found")); + } public MEIndicatorDTO createMEIndicator(final MEIndicatorDTO indicatorRequest) { Session session = PersistenceManager.getSession(); diff --git a/amp/src/main/java/org/digijava/module/message/jobs/AmpDonorFundingJob.java b/amp/src/main/java/org/digijava/module/message/jobs/AmpDonorFundingJob.java index 3e28ec4261f..84c625badb7 100644 --- a/amp/src/main/java/org/digijava/module/message/jobs/AmpDonorFundingJob.java +++ b/amp/src/main/java/org/digijava/module/message/jobs/AmpDonorFundingJob.java @@ -485,7 +485,7 @@ public static void sendReportsToServer(List ampDashboardFundin // Convert to JSON using a JSON library (e.g., Gson) Gson gson = new Gson(); String jsonData = gson.toJson(submissionData); - logger.info("JSON data: " + jsonData); +// logger.info("JSON data: " + jsonData); // Get the output stream of the connection try (OutputStream os = connection.getOutputStream()) { diff --git a/amp/src/main/resources/xmlpatches/4.0/AMP-30885-Data-Importer-Menu-Items-v3.xml b/amp/src/main/resources/xmlpatches/4.0/AMP-30885-Data-Importer-Menu-Items-v3.xml deleted file mode 100644 index fca110892ba..00000000000 --- a/amp/src/main/resources/xmlpatches/4.0/AMP-30885-Data-Importer-Menu-Items-v3.xml +++ /dev/null @@ -1,56 +0,0 @@ - - - AMP-30885 - menu - bmokandu - Add menu entry for Data Importer - - - - diff --git a/amp/src/main/resources/xmlpatches/4.0/AMP-30885-Data-Importer-Menu-Items-v4.xml b/amp/src/main/resources/xmlpatches/4.0/AMP-30885-Data-Importer-Menu-Items-v4.xml new file mode 100644 index 00000000000..91a3ed4bea8 --- /dev/null +++ b/amp/src/main/resources/xmlpatches/4.0/AMP-30885-Data-Importer-Menu-Items-v4.xml @@ -0,0 +1,67 @@ + + + AMP-30885 + menu + bmokandu + Add menu entry for Data Importer + + + + From 4127921c47ae4ec75783773e1576ddf8eac81ef6 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Fri, 13 Feb 2026 18:54:03 +0300 Subject: [PATCH 03/59] Add indicator configuration for import --- .../manager/IndicatorManagerService.java | 15 ++ .../aim/action/dataimporter/DataImporter.java | 15 ++ .../action/dataimporter/ExcelImporter.java | 11 +- .../dataimporter/util/ImporterUtil.java | 210 ++++++++++++++++-- 4 files changed, 236 insertions(+), 15 deletions(-) diff --git a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/indicator/manager/IndicatorManagerService.java b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/indicator/manager/IndicatorManagerService.java index c043e7e09f1..eefcfb1ee27 100644 --- a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/indicator/manager/IndicatorManagerService.java +++ b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/indicator/manager/IndicatorManagerService.java @@ -107,6 +107,21 @@ public MEIndicatorDTO getMeIndicatorByNameAndProgramName(String name, String pro ApiError.toError("Indicator with name " + name + " and program name " + programName + " not found")); } + /** + * Returns the indicator by name and optional program name, or null if not found. + * Use this when you need to look up an indicator without throwing (e.g. data import). + */ + public MEIndicatorDTO getMeIndicatorByNameAndProgramNameOptional(String name, String programName) { + if (name == null || name.trim().isEmpty()) { + return null; + } + try { + return getMeIndicatorByNameAndProgramName(name.trim(), programName != null && !programName.trim().isEmpty() ? programName.trim() : null); + } catch (ApiRuntimeException e) { + return null; + } + } + public MEIndicatorDTO createMEIndicator(final MEIndicatorDTO indicatorRequest) { Session session = PersistenceManager.getSession(); AmpIndicator indicator = new AmpIndicator(); diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java index 4aa69703e6d..e08a38fc29a 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java @@ -494,6 +494,21 @@ private List getEntityFieldsInfo() { fieldsInfos.add("Component Name"); fieldsInfos.add("Component Code"); fieldsInfos.add("Beneficiary Agency"); + // Indicator columns for M&E import + fieldsInfos.add("Indicator Name"); + fieldsInfos.add("Program Name"); + fieldsInfos.add("Location"); + fieldsInfos.add("Original Base Value"); + fieldsInfos.add("Original Base Value Date"); + fieldsInfos.add("Revised Base Value"); + fieldsInfos.add("Revised Base Value Date"); + fieldsInfos.add("Original Target Value"); + fieldsInfos.add("Original Target Value Date"); + fieldsInfos.add("Revised Target Value"); + fieldsInfos.add("Revised Target Value Date"); + fieldsInfos.add("Actual Value"); + fieldsInfos.add("Actual Value Date"); + fieldsInfos.add("Unit of Measure"); return fieldsInfos.stream().sorted().collect(Collectors.toList()); } diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java index 1a9514fd660..74de4d1b243 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java @@ -31,6 +31,7 @@ import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.getColumnIndexByName; import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.getKey; import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.getStringValueFromCell; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.addIndicatorDataToActivity; import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.importTheData; import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.setAFundingItemForExcel; import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.setStatus; @@ -278,8 +279,14 @@ public static void processBatch(List batch,Sheet sheet, HttpServletRequest } - importTheData(importDataModel, session, importedProject, componentName, componentCode, responsibleOrgId, fundings, existing); - + Long activityId = importTheData(importDataModel, session, importedProject, componentName, componentCode, responsibleOrgId, fundings, existing); + if (activityId != null && config.containsValue("Indicator Name")) { + try { + addIndicatorDataToActivity(activityId, row, sheet, config, session); + } catch (Exception e) { + logger.error("Failed to add indicator data for activity " + activityId, e); + } + } } } } diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java index 357fc6c0c79..3d178e2ddf6 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java @@ -13,6 +13,8 @@ import org.digijava.kernel.ampapi.endpoints.activity.ActivityInterchangeUtils; import org.digijava.kernel.ampapi.endpoints.activity.dto.ActivitySummary; import org.digijava.kernel.ampapi.endpoints.common.JsonApiResponse; +import org.digijava.kernel.ampapi.endpoints.indicator.manager.IndicatorManagerService; +import org.digijava.kernel.ampapi.endpoints.indicator.manager.MEIndicatorDTO; import org.digijava.kernel.persistence.PersistenceManager; import org.digijava.module.aim.action.dataimporter.dbentity.ImportStatus; import org.digijava.module.aim.action.dataimporter.dbentity.ImportedProject; @@ -20,6 +22,7 @@ import org.digijava.module.aim.action.dataimporter.model.*; import org.digijava.module.aim.dbentity.*; import org.digijava.module.aim.util.CurrencyUtil; +import org.digijava.module.aim.util.ProgramUtil; import org.digijava.module.categorymanager.dbentity.AmpCategoryValue; import org.digijava.module.categorymanager.util.CategoryConstants; import org.hibernate.Query; @@ -571,14 +574,27 @@ public static AmpActivityVersion existingActivity(String projectTitle, String pr if (!session.isOpen()) { session = PersistenceManager.getRequestDBSession(); } - String hql = "SELECT a FROM " + AmpActivityVersion.class.getName() + " a " + - "WHERE a.name = :name"; - Query query = session.createQuery(hql); - query.setCacheable(true); - query.setParameter("name", projectTitle, StringType.INSTANCE); -// query.setString("projectCode", projectCode); - List ampActivityVersions = query.list(); - return !ampActivityVersions.isEmpty() ? ampActivityVersions.get(ampActivityVersions.size() - 1) : null; + // Prefer project code if provided + if (projectCode != null && !projectCode.trim().isEmpty()) { + String hqlByCode = "SELECT a FROM " + AmpActivityVersion.class.getName() + " a WHERE a.projectCode = :projectCode"; + Query queryByCode = session.createQuery(hqlByCode); + queryByCode.setCacheable(true); + queryByCode.setParameter("projectCode", projectCode.trim(), StringType.INSTANCE); + List byCode = queryByCode.list(); + if (!byCode.isEmpty()) { + return byCode.get(byCode.size() - 1); + } + } + // Fall back to project title (name) + if (projectTitle != null && !projectTitle.trim().isEmpty()) { + String hql = "SELECT a FROM " + AmpActivityVersion.class.getName() + " a WHERE a.name = :name"; + Query query = session.createQuery(hql); + query.setCacheable(true); + query.setParameter("name", projectTitle.trim(), StringType.INSTANCE); + List ampActivityVersions = query.list(); + return !ampActivityVersions.isEmpty() ? ampActivityVersions.get(ampActivityVersions.size() - 1) : null; + } + return null; } public static void setStatus(ImportDataModel importDataModel) { @@ -587,7 +603,8 @@ public static void setStatus(ImportDataModel importDataModel) { importDataModel.setApproval_status(ApprovalStatus.started.getId()); } - public static void importTheData(ImportDataModel importDataModel, Session session, ImportedProject importedProject, String componentName, String componentCode, Long responsibleOrgId, List fundings, AmpActivityVersion existing) throws JsonProcessingException { + /** @return activity ID on success, null on skip or failure */ + public static Long importTheData(ImportDataModel importDataModel, Session session, ImportedProject importedProject, String componentName, String componentCode, Long responsibleOrgId, List fundings, AmpActivityVersion existing) throws JsonProcessingException { if (!session.isOpen()) { session = PersistenceManager.getRequestDBSession(); } @@ -605,7 +622,7 @@ public static void importTheData(ImportDataModel importDataModel, Session sessio if (importDataModel.getProject_title().trim().isEmpty() && importDataModel.getProject_code().trim().isEmpty()) { logger.info("Project title and code are empty. Skipping import"); importedProject.setImportStatus(ImportStatus.SKIPPED); - return; + return null; } if (existing == null) { logger.info("New activity"); @@ -627,20 +644,19 @@ public static void importTheData(ImportDataModel importDataModel, Session sessio }); response = ActivityInterchangeUtils.importActivity(map, true, rules, "activity/update"); } + Long activityId = null; if (response != null) { if (!response.getErrors().isEmpty()) { importedProject.setImportStatus(ImportStatus.FAILED); } else { importedProject.setImportStatus(ImportStatus.SUCCESS); + activityId = existing != null ? existing.getAmpActivityId() : (Long) response.getContent().getAmpActivityId(); logger.info("Successfully imported the project. Now adding component if present"); logger.info("--------------------------------"); logger.info("Component name at start: " + componentName); if (componentName != null && !componentName.isEmpty()) { addComponentsAndProjectCode(response, componentName, componentCode, responsibleOrgId, fundings, importDataModel.getProject_code()); } -// logger.info("Updating expenditures ................"); -// updateExpendituresIfAny(response); - } } @@ -653,6 +669,7 @@ public static void importTheData(ImportDataModel importDataModel, Session sessio session.flush(); logger.info("Imported project: " + importedProject); + return activityId; } private static void updateFundingOrgsAndSectorsWithAlreadyExisting(AmpActivityVersion ampActivityVersion, ImportDataModel importDataModel) { @@ -1223,4 +1240,171 @@ public static int getColumnIndexByName(Sheet sheet, String columnName) { } } + + /** Parse date from Excel cell or string; returns today if null/empty/invalid. */ + public static Date parseDateDefaultToday(Row row, Sheet sheet, Map config, String columnName) { + String key = getKey(config, columnName); + if (key == null) return Date.from(LocalDate.now().atStartOfDay(ZoneId.systemDefault()).toInstant()); + int col = getColumnIndexByName(sheet, key); + if (col < 0) return Date.from(LocalDate.now().atStartOfDay(ZoneId.systemDefault()).toInstant()); + Cell cell = row.getCell(col); + if (cell == null) return Date.from(LocalDate.now().atStartOfDay(ZoneId.systemDefault()).toInstant()); + String dateStr = extractDateFromStringCell(cell); + if (dateStr == null || dateStr.isEmpty()) return Date.from(LocalDate.now().atStartOfDay(ZoneId.systemDefault()).toInstant()); + try { + return java.sql.Date.valueOf(dateStr); + } catch (Exception e) { + return Date.from(LocalDate.now().atStartOfDay(ZoneId.systemDefault()).toInstant()); + } + } + + /** Add indicator data to an activity from the current row. Called after importTheData when indicator columns are mapped. */ + public static void addIndicatorDataToActivity(Long activityId, Row row, Sheet sheet, Map config, Session session) { + if (activityId == null || config == null || row == null || sheet == null) return; + if (getKey(config, "Indicator Name") == null || getKey(config, "Location") == null || getKey(config, "Actual Value") == null) { + return; + } + String indicatorName = getCellValueByConfig(row, sheet, config, "Indicator Name"); + String locationName = getCellValueByConfig(row, sheet, config, "Location"); + if (indicatorName == null || indicatorName.trim().isEmpty() || locationName == null || locationName.trim().isEmpty()) { + return; + } + indicatorName = indicatorName.trim(); + locationName = locationName.trim(); + String programName = getCellValueByConfig(row, sheet, config, "Program Name"); + if (programName != null) programName = programName.trim(); + + IndicatorManagerService indicatorService = new IndicatorManagerService(); + MEIndicatorDTO indicatorDto = indicatorService.getMeIndicatorByNameAndProgramNameOptional(indicatorName, (programName == null || programName.isEmpty()) ? null : programName); + AmpIndicator indicator; + if (indicatorDto == null) { + MEIndicatorDTO createDto = new MEIndicatorDTO(); + createDto.setName(indicatorName); + createDto.setCode(indicatorName + "_" + System.currentTimeMillis()); + createDto.setCreationDate(new Date()); + createDto.setAscending(true); + createDto.setSectorIds(new ArrayList<>()); + if (programName != null && !programName.isEmpty()) { + try { + AmpTheme theme = ProgramUtil.getTheme(programName); + if (theme != null && theme.getAmpThemeId() != null) createDto.setProgramId(theme.getAmpThemeId()); + } catch (Exception e) { + logger.warn("Could not resolve program by name: " + programName, e); + } + } + try { + indicatorDto = indicatorService.createMEIndicator(createDto); + } catch (Exception e) { + logger.error("Failed to create indicator: " + indicatorName, e); + return; + } + indicator = (AmpIndicator) session.get(AmpIndicator.class, indicatorDto.getId()); + } else { + indicator = (AmpIndicator) session.get(AmpIndicator.class, indicatorDto.getId()); + } + if (indicator == null) return; + + AmpActivityVersion activity = (AmpActivityVersion) session.get(AmpActivityVersion.class, activityId); + if (activity == null) return; + AmpActivityLocation activityLocation = null; + if (activity.getLocations() != null) { + for (AmpActivityLocation aal : activity.getLocations()) { + if (aal.getLocation() != null && locationName.equalsIgnoreCase(aal.getLocation().getName())) { + activityLocation = aal; + break; + } + } + } + + IndicatorActivity ia = new IndicatorActivity(); + ia.setActivity(activity); + ia.setIndicator(indicator); + if (activityLocation != null) ia.setActivityLocation(activityLocation); + + AmpIndicatorGlobalValue existingBase = indicator.getBaseValue(); + + double origBase = parseDoubleFromConfig(row, sheet, config, "Original Base Value"); + boolean hasOrigBase = getKey(config, "Original Base Value") != null && !Double.isNaN(origBase); + double revBase = parseDoubleFromConfig(row, sheet, config, "Revised Base Value"); + boolean hasRevBase = getKey(config, "Revised Base Value") != null && !Double.isNaN(revBase); + double origTarget = parseDoubleFromConfig(row, sheet, config, "Original Target Value"); + double revTarget = parseDoubleFromConfig(row, sheet, config, "Revised Target Value"); + double actualVal = parseDoubleFromConfig(row, sheet, config, "Actual Value"); + if (Double.isNaN(actualVal)) actualVal = 0.0; + + Date origBaseDate = parseDateDefaultToday(row, sheet, config, "Original Base Value Date"); + Date revBaseDate = parseDateDefaultToday(row, sheet, config, "Revised Base Value Date"); + Date origTargetDate = parseDateDefaultToday(row, sheet, config, "Original Target Value Date"); + Date revTargetDate = parseDateDefaultToday(row, sheet, config, "Revised Target Value Date"); + Date actualDate = parseDateDefaultToday(row, sheet, config, "Actual Value Date"); + + double baseOrigVal = hasOrigBase ? origBase : (existingBase != null && existingBase.getOriginalValue() != null ? existingBase.getOriginalValue() : 0.0); + double baseRevVal = hasRevBase ? revBase : (existingBase != null && existingBase.getRevisedValue() != null ? existingBase.getRevisedValue() : 0.0); + double targetOrigVal = Double.isNaN(origTarget) ? 0.0 : origTarget; + double targetRevVal = Double.isNaN(revTarget) ? 0.0 : revTarget; + + Set values = new HashSet<>(); + if (getKey(config, "Original Base Value") != null || getKey(config, "Revised Base Value") != null || existingBase != null) { + AmpIndicatorValue baseOrig = new AmpIndicatorValue(AmpIndicatorValue.BASE); + baseOrig.setValue(baseOrigVal); + baseOrig.setValueDate(origBaseDate); + baseOrig.setIndicatorConnection(ia); + values.add(baseOrig); + AmpIndicatorValue baseRev = new AmpIndicatorValue(AmpIndicatorValue.REVISED); + baseRev.setValue(baseRevVal); + baseRev.setValueDate(revBaseDate); + baseRev.setIndicatorConnection(ia); + values.add(baseRev); + } + if (getKey(config, "Original Target Value") != null || getKey(config, "Revised Target Value") != null) { + AmpIndicatorValue tOrig = new AmpIndicatorValue(AmpIndicatorValue.TARGET); + tOrig.setValue(targetOrigVal); + tOrig.setValueDate(origTargetDate); + tOrig.setIndicatorConnection(ia); + values.add(tOrig); + AmpIndicatorValue tRev = new AmpIndicatorValue(AmpIndicatorValue.TARGET); + tRev.setValue(targetRevVal); + tRev.setValueDate(revTargetDate); + tRev.setIndicatorConnection(ia); + values.add(tRev); + } + AmpIndicatorValue actual = new AmpIndicatorValue(AmpIndicatorValue.ACTUAL); + actual.setValue(actualVal); + actual.setValueDate(actualDate); + actual.setIndicatorConnection(ia); + String unit = getCellValueByConfig(row, sheet, config, "Unit of Measure"); + if (unit != null && !unit.trim().isEmpty()) actual.setComment("Unit: " + unit.trim()); + values.add(actual); + + ia.setValues(values); + if (activity.getIndicators() == null) activity.setIndicators(new HashSet<>()); + activity.getIndicators().add(ia); + session.save(ia); + session.flush(); + } + + private static String getCellValueByConfig(Row row, Sheet sheet, Map config, String fieldName) { + String key = getKey(config, fieldName); + if (key == null) return null; + int col = getColumnIndexByName(sheet, key); + if (col < 0) return null; + return getStringValueFromCell(row.getCell(col), true); + } + + private static double parseDoubleFromConfig(Row row, Sheet sheet, Map config, String fieldName) { + String key = getKey(config, fieldName); + if (key == null) return Double.NaN; + int col = getColumnIndexByName(sheet, key); + if (col < 0) return Double.NaN; + Cell cell = row.getCell(col); + if (cell == null) return Double.NaN; + try { + if (cell.getCellType() == Cell.CELL_TYPE_NUMERIC) return cell.getNumericCellValue(); + String s = getStringValueFromCell(cell, true); + if (s == null || s.trim().isEmpty()) return Double.NaN; + return Double.parseDouble(s.trim()); + } catch (Exception e) { + return Double.NaN; + } + } } From 3fba4c6969229aad1be3826f47c34b7b097e4d60 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Sat, 14 Feb 2026 22:20:52 +0300 Subject: [PATCH 04/59] Add program if missing and attach to activity; including percentages --- .../org/dgfoundation/amp/ar/ArConstants.java | 3 +- .../dataimporter/util/ImporterUtil.java | 102 ++++++++++++++++-- 2 files changed, 93 insertions(+), 12 deletions(-) diff --git a/amp/src/main/java/org/dgfoundation/amp/ar/ArConstants.java b/amp/src/main/java/org/dgfoundation/amp/ar/ArConstants.java index 43bcfe05720..d7b6ef38acc 100644 --- a/amp/src/main/java/org/dgfoundation/amp/ar/ArConstants.java +++ b/amp/src/main/java/org/dgfoundation/amp/ar/ArConstants.java @@ -387,8 +387,7 @@ public final class ArConstants { // public final static String EXECUTING_AGENCY_PERCENTAGE="Eexecuting Agency Percentage"; - //burkina -// public final static String PROGRAM_PERCENTAGE="Program Percentage"; + public final static String PROGRAM_PERCENTAGE="Program Percentage"; diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java index 3d178e2ddf6..73d9ea26f62 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java @@ -9,6 +9,7 @@ import org.apache.poi.ss.usermodel.DateUtil; import org.apache.poi.ss.usermodel.Row; import org.apache.poi.ss.usermodel.Sheet; +import org.dgfoundation.amp.ar.ArConstants; import org.digijava.kernel.ampapi.endpoints.activity.ActivityImportRules; import org.digijava.kernel.ampapi.endpoints.activity.ActivityInterchangeUtils; import org.digijava.kernel.ampapi.endpoints.activity.dto.ActivitySummary; @@ -22,6 +23,7 @@ import org.digijava.module.aim.action.dataimporter.model.*; import org.digijava.module.aim.dbentity.*; import org.digijava.module.aim.util.CurrencyUtil; +import org.digijava.module.aim.util.FeaturesUtil; import org.digijava.module.aim.util.ProgramUtil; import org.digijava.module.categorymanager.dbentity.AmpCategoryValue; import org.digijava.module.categorymanager.util.CategoryConstants; @@ -1274,6 +1276,15 @@ public static void addIndicatorDataToActivity(Long activityId, Row row, Sheet sh String programName = getCellValueByConfig(row, sheet, config, "Program Name"); if (programName != null) programName = programName.trim(); + AmpTheme programTheme = null; + if (programName != null && !programName.isEmpty()) { + try { + programTheme = getOrCreateProgramByName(programName, session); + } catch (Exception e) { + logger.warn("Could not resolve or create program by name: " + programName, e); + } + } + IndicatorManagerService indicatorService = new IndicatorManagerService(); MEIndicatorDTO indicatorDto = indicatorService.getMeIndicatorByNameAndProgramNameOptional(indicatorName, (programName == null || programName.isEmpty()) ? null : programName); AmpIndicator indicator; @@ -1284,13 +1295,8 @@ public static void addIndicatorDataToActivity(Long activityId, Row row, Sheet sh createDto.setCreationDate(new Date()); createDto.setAscending(true); createDto.setSectorIds(new ArrayList<>()); - if (programName != null && !programName.isEmpty()) { - try { - AmpTheme theme = ProgramUtil.getTheme(programName); - if (theme != null && theme.getAmpThemeId() != null) createDto.setProgramId(theme.getAmpThemeId()); - } catch (Exception e) { - logger.warn("Could not resolve program by name: " + programName, e); - } + if (programTheme != null && programTheme.getAmpThemeId() != null) { + createDto.setProgramId(programTheme.getAmpThemeId()); } try { indicatorDto = indicatorService.createMEIndicator(createDto); @@ -1298,14 +1304,19 @@ public static void addIndicatorDataToActivity(Long activityId, Row row, Sheet sh logger.error("Failed to create indicator: " + indicatorName, e); return; } - indicator = (AmpIndicator) session.get(AmpIndicator.class, indicatorDto.getId()); + indicator = session.get(AmpIndicator.class, indicatorDto.getId()); } else { - indicator = (AmpIndicator) session.get(AmpIndicator.class, indicatorDto.getId()); + indicator = session.get(AmpIndicator.class, indicatorDto.getId()); } if (indicator == null) return; - AmpActivityVersion activity = (AmpActivityVersion) session.get(AmpActivityVersion.class, activityId); + AmpActivityVersion activity = session.get(AmpActivityVersion.class, activityId); if (activity == null) return; + + if (programTheme != null) { + addProgramToActivityIfMissing(activity, programTheme, session); + } + AmpActivityLocation activityLocation = null; if (activity.getLocations() != null) { for (AmpActivityLocation aal : activity.getLocations()) { @@ -1383,6 +1394,77 @@ public static void addIndicatorDataToActivity(Long activityId, Row row, Sheet sh session.flush(); } + /** + * Adds the program (theme) to the activity's programs if not already present. + * When the Program Percentage field is enabled, percentages are recalculated and + * divided evenly among all activity programs (including the one just added). + */ + public static void addProgramToActivityIfMissing(AmpActivityVersion activity, AmpTheme program, Session session) { + if (activity == null || program == null) return; + Set actPrograms = activity.getActPrograms(); + if (actPrograms == null) { + actPrograms = new HashSet<>(); + activity.setActPrograms(actPrograms); + } + for (AmpActivityProgram ap : actPrograms) { + if (ap.getProgram() != null && program.getAmpThemeId() != null + && program.getAmpThemeId().equals(ap.getProgram().getAmpThemeId())) { + return; + } + } + AmpActivityProgram activityProgram = new AmpActivityProgram(); + activityProgram.setActivity(activity); + activityProgram.setProgram(program); + activityProgram.setProgramPercentage(100f); + actPrograms.add(activityProgram); + session.save(activityProgram); + + // If Program Percentage field is enabled, distribute 100% evenly among all programs + boolean percentageEnabled = false; + try { + percentageEnabled = FeaturesUtil.isVisibleField(ArConstants.PROGRAM_PERCENTAGE); + } catch (Exception e) { + // No request/session (e.g. batch) – skip percentage redistribution + logger.error("Could not determine if Program Percentage field is enabled; skipping percentage redistribution", e); + } + if (percentageEnabled && !actPrograms.isEmpty()) { + List list = new ArrayList<>(actPrograms); + int n = list.size(); + Map percentages = divide100(n); + for (int i = 0; i < n; i++) { + list.get(i).setProgramPercentage(percentages.get(i)); + } + } + } + + /** + * Returns the program (theme) by name, or creates a new root-level program if it does not exist. + * @param programName program name (must be non-empty) + * @param session current session (used for create and flush) + * @return AmpTheme or null if programName is null/empty or creation fails + */ + public static AmpTheme getOrCreateProgramByName(String programName, Session session) { + if (programName == null || programName.trim().isEmpty()) return null; + programName = programName.trim(); + AmpTheme theme = ProgramUtil.getTheme(programName); + if (theme != null) return theme; + try { + AmpTheme newTheme = new AmpTheme(); + newTheme.setName(programName); + String code = programName.replaceAll("[^a-zA-Z0-9_-]", "_").replaceAll("_+", "_").trim(); + if (code.length() > 45) code = code.substring(0, 45); + newTheme.setThemeCode("IMP_" + code + "_" + System.currentTimeMillis()); + newTheme.setIndlevel(0); + newTheme.setParentThemeId(null); + session.save(newTheme); + session.flush(); + return newTheme; + } catch (Exception e) { + logger.warn("Failed to create program: " + programName, e); + return null; + } + } + private static String getCellValueByConfig(Row row, Sheet sheet, Map config, String fieldName) { String key = getKey(config, fieldName); if (key == null) return null; From d2a3bcbb3f40c2b27bd537082ed1051403be1119 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Sun, 15 Feb 2026 19:32:47 +0300 Subject: [PATCH 05/59] Use constants for common strings Add MeasureType field config --- .../aim/action/dataimporter/DataImporter.java | 97 ++++++------- .../action/dataimporter/ExcelImporter.java | 88 +++++++----- .../action/dataimporter/TxtDataImporter.java | 83 ++++++----- .../dataimporter/util/ImporterConstants.java | 74 ++++++++++ .../dataimporter/util/ImporterUtil.java | 130 ++++++++++++------ 5 files changed, 317 insertions(+), 155 deletions(-) create mode 100644 amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterConstants.java diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java index e08a38fc29a..9b2fd297019 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java @@ -49,6 +49,8 @@ import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.isFileContentValid; import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.isFileReadable; import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.removeMapItem; + +import org.digijava.module.aim.action.dataimporter.util.ImporterConstants; import org.digijava.module.aim.form.DataImporterForm; import org.hibernate.Query; import org.hibernate.Session; @@ -287,7 +289,7 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet } } } - if (dataImporterForm.getColumnPairs().isEmpty() ||(!dataImporterForm.getColumnPairs().containsValue("Project Title") && !dataImporterForm.getColumnPairs().containsValue("Project Code"))) { + if (dataImporterForm.getColumnPairs().isEmpty() ||(!dataImporterForm.getColumnPairs().containsValue(ImporterConstants.PROJECT_TITLE) && !dataImporterForm.getColumnPairs().containsValue(ImporterConstants.PROJECT_CODE))) { response.setHeader("errorMessage", "You must have at least the 'Project Title' or 'Project Code' column in your config."); response.setStatus(400); @@ -462,53 +464,54 @@ public static void saveImportConfig(HttpServletRequest request, String fileName, private List getEntityFieldsInfo() { List fieldsInfos = new ArrayList<>(); - fieldsInfos.add("Project Title"); - fieldsInfos.add("Project Code"); - fieldsInfos.add("Objective"); - fieldsInfos.add("Project Description"); - fieldsInfos.add("Primary Sector"); - fieldsInfos.add("Secondary Sector"); - fieldsInfos.add("Project Location"); - fieldsInfos.add("Project Start Date"); - fieldsInfos.add("Project End Date"); - fieldsInfos.add("Donor Agency"); - fieldsInfos.add("Exchange Rate"); - fieldsInfos.add("Donor Agency Code"); - fieldsInfos.add("Responsible Organization"); - fieldsInfos.add("Responsible Organization Code"); - fieldsInfos.add("Executing Agency"); - fieldsInfos.add("Implementing Agency"); - fieldsInfos.add("Actual Disbursement"); - fieldsInfos.add("Actual Commitment"); - fieldsInfos.add("Actual Expenditure"); - fieldsInfos.add("Planned Disbursement"); - fieldsInfos.add("Planned Commitment"); - fieldsInfos.add("Planned Expenditure"); - fieldsInfos.add("Funding Item"); - fieldsInfos.add("Transaction Date"); - fieldsInfos.add("Financing Instrument"); - fieldsInfos.add("Type Of Assistance"); - fieldsInfos.add("Secondary Subsector"); - fieldsInfos.add("Primary Subsector"); - fieldsInfos.add("Currency"); - fieldsInfos.add("Component Name"); - fieldsInfos.add("Component Code"); - fieldsInfos.add("Beneficiary Agency"); + fieldsInfos.add(ImporterConstants.PROJECT_TITLE); + fieldsInfos.add(ImporterConstants.PROJECT_CODE); + fieldsInfos.add(ImporterConstants.OBJECTIVE); + fieldsInfos.add(ImporterConstants.PROJECT_DESCRIPTION); + fieldsInfos.add(ImporterConstants.PRIMARY_SECTOR); + fieldsInfos.add(ImporterConstants.SECONDARY_SECTOR); + fieldsInfos.add(ImporterConstants.PROJECT_LOCATION); + fieldsInfos.add(ImporterConstants.PROJECT_START_DATE); + fieldsInfos.add(ImporterConstants.PROJECT_END_DATE); + fieldsInfos.add(ImporterConstants.DONOR_AGENCY); + fieldsInfos.add(ImporterConstants.EXCHANGE_RATE); + fieldsInfos.add(ImporterConstants.DONOR_AGENCY_CODE); + fieldsInfos.add(ImporterConstants.RESPONSIBLE_ORGANIZATION); + fieldsInfos.add(ImporterConstants.RESPONSIBLE_ORGANIZATION_CODE); + fieldsInfos.add(ImporterConstants.EXECUTING_AGENCY); + fieldsInfos.add(ImporterConstants.IMPLEMENTING_AGENCY); + fieldsInfos.add(ImporterConstants.ACTUAL_DISBURSEMENT); + fieldsInfos.add(ImporterConstants.ACTUAL_COMMITMENT); + fieldsInfos.add(ImporterConstants.ACTUAL_EXPENDITURE); + fieldsInfos.add(ImporterConstants.PLANNED_DISBURSEMENT); + fieldsInfos.add(ImporterConstants.PLANNED_COMMITMENT); + fieldsInfos.add(ImporterConstants.PLANNED_EXPENDITURE); + fieldsInfos.add(ImporterConstants.TRANSACTION_AMOUNT); + fieldsInfos.add(ImporterConstants.MEASURE_TYPE); + fieldsInfos.add(ImporterConstants.TRANSACTION_DATE); + fieldsInfos.add(ImporterConstants.FINANCING_INSTRUMENT); + fieldsInfos.add(ImporterConstants.TYPE_OF_ASSISTANCE); + fieldsInfos.add(ImporterConstants.SECONDARY_SUBSECTOR); + fieldsInfos.add(ImporterConstants.PRIMARY_SUBSECTOR); + fieldsInfos.add(ImporterConstants.CURRENCY); + fieldsInfos.add(ImporterConstants.COMPONENT_NAME); + fieldsInfos.add(ImporterConstants.COMPONENT_CODE); + fieldsInfos.add(ImporterConstants.BENEFICIARY_AGENCY); // Indicator columns for M&E import - fieldsInfos.add("Indicator Name"); - fieldsInfos.add("Program Name"); - fieldsInfos.add("Location"); - fieldsInfos.add("Original Base Value"); - fieldsInfos.add("Original Base Value Date"); - fieldsInfos.add("Revised Base Value"); - fieldsInfos.add("Revised Base Value Date"); - fieldsInfos.add("Original Target Value"); - fieldsInfos.add("Original Target Value Date"); - fieldsInfos.add("Revised Target Value"); - fieldsInfos.add("Revised Target Value Date"); - fieldsInfos.add("Actual Value"); - fieldsInfos.add("Actual Value Date"); - fieldsInfos.add("Unit of Measure"); + fieldsInfos.add(ImporterConstants.INDICATOR_NAME); + fieldsInfos.add(ImporterConstants.PROGRAM_NAME); + fieldsInfos.add(ImporterConstants.LOCATION); + fieldsInfos.add(ImporterConstants.ORIGINAL_BASE_VALUE); + fieldsInfos.add(ImporterConstants.ORIGINAL_BASE_VALUE_DATE); + fieldsInfos.add(ImporterConstants.REVISED_BASE_VALUE); + fieldsInfos.add(ImporterConstants.REVISED_BASE_VALUE_DATE); + fieldsInfos.add(ImporterConstants.ORIGINAL_TARGET_VALUE); + fieldsInfos.add(ImporterConstants.ORIGINAL_TARGET_VALUE_DATE); + fieldsInfos.add(ImporterConstants.REVISED_TARGET_VALUE); + fieldsInfos.add(ImporterConstants.REVISED_TARGET_VALUE_DATE); + fieldsInfos.add(ImporterConstants.ACTUAL_VALUE); + fieldsInfos.add(ImporterConstants.ACTUAL_VALUE_DATE); + fieldsInfos.add(ImporterConstants.UNIT_OF_MEASURE); return fieldsInfos.stream().sorted().collect(Collectors.toList()); } diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java index 74de4d1b243..294b8557c29 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java @@ -26,11 +26,14 @@ import org.digijava.module.aim.action.dataimporter.dbentity.ImportedProject; import org.digijava.module.aim.action.dataimporter.model.Funding; import org.digijava.module.aim.action.dataimporter.model.ImportDataModel; +import org.digijava.module.aim.action.dataimporter.util.ImporterConstants; import org.digijava.module.aim.action.dataimporter.util.ImportedFileUtil; +import org.digijava.module.aim.action.dataimporter.util.ImporterUtil; import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.existingActivity; import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.getColumnIndexByName; import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.getKey; import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.getStringValueFromCell; +import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.parseMeasureType; import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.addIndicatorDataToActivity; import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.importTheData; import static org.digijava.module.aim.action.dataimporter.util.ImporterUtil.setAFundingItemForExcel; @@ -172,36 +175,36 @@ public static void processBatch(List batch,Sheet sheet, HttpServletRequest importDataModel.setCreation_date(now.format(formatter)); setStatus(importDataModel); - int componentCodeColumn = getColumnIndexByName(sheet, getKey(config, "Component Code")); + int componentCodeColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.COMPONENT_CODE)); String componentCode = componentCodeColumn >= 0 ? getStringValueFromCell(row.getCell(componentCodeColumn),true) : null; - int componentNameColumn = getColumnIndexByName(sheet, getKey(config, "Component Name")); + int componentNameColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.COMPONENT_NAME)); String componentName = componentNameColumn >= 0 ? getStringValueFromCell(row.getCell(componentNameColumn),true): null; - int donorAgencyCodeColumn = getColumnIndexByName(sheet, getKey(config, "Donor Agency Code")); + int donorAgencyCodeColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.DONOR_AGENCY_CODE)); String donorAgencyCode = donorAgencyCodeColumn >= 0 ? getStringValueFromCell(row.getCell(donorAgencyCodeColumn),true) : null; - int responsibleOrgCodeColumn = getColumnIndexByName(sheet, getKey(config, "Responsible Organization Code")); + int responsibleOrgCodeColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.RESPONSIBLE_ORGANIZATION_CODE)); String responsibleOrgCode = responsibleOrgCodeColumn >= 0 ? getStringValueFromCell(row.getCell(responsibleOrgCodeColumn),true) : null; - int primarySubSectorColumn = getColumnIndexByName(sheet, getKey(config, "Primary Subsector")); + int primarySubSectorColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.PRIMARY_SUBSECTOR)); String primarySubSector = primarySubSectorColumn >= 0 ? getStringValueFromCell(row.getCell(primarySubSectorColumn),true) : null; - int secondarySubSectorColumn = getColumnIndexByName(sheet, getKey(config, "Secondary Subsector")); + int secondarySubSectorColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.SECONDARY_SUBSECTOR)); String secondarySubSector = secondarySubSectorColumn >= 0 ? getStringValueFromCell(row.getCell(secondarySubSectorColumn),true) : null; - int projectCodeColumn = getColumnIndexByName(sheet, getKey(config, "Project Code")); + int projectCodeColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.PROJECT_CODE)); String projectCode = projectCodeColumn >= 0 ? getStringValueFromCell(row.getCell(projectCodeColumn),false) : ""; importDataModel.setProject_code(projectCode); - int projectTitleColumn = getColumnIndexByName(sheet, getKey(config, "Project Title")); + int projectTitleColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.PROJECT_TITLE)); String projectTitle = projectTitleColumn >= 0 ? getStringValueFromCell(row.getCell(projectTitleColumn),false) : ""; importDataModel.setProject_title(projectTitle); - int objectiveColumn = getColumnIndexByName(sheet, getKey(config, "Objective")); + int objectiveColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.OBJECTIVE)); String objective = objectiveColumn >= 0 ? getStringValueFromCell(row.getCell(objectiveColumn),false) : null; importDataModel.setObjective(objective); - int projectDescColumn = getColumnIndexByName(sheet, getKey(config, "Project Description")); + int projectDescColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.PROJECT_DESCRIPTION)); String projectDesc = projectDescColumn >= 0 ? getStringValueFromCell(row.getCell(projectDescColumn),false) : null; importDataModel.setDescription(projectDesc); @@ -224,47 +227,62 @@ public static void processBatch(List batch,Sheet sheet, HttpServletRequest if (columnIndex >= 0) { Cell cell = row.getCell(columnIndex); switch (entry.getValue()) { - case "Project Location": + case ImporterConstants.PROJECT_LOCATION: updateLocations(importDataModel,Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(),session); break; - case "Primary Sector": + case ImporterConstants.PRIMARY_SECTOR: updateSectors(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), session, true, primarySubSector); break; - case "Secondary Sector": + case ImporterConstants.SECONDARY_SECTOR: updateSectors(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), session, false, secondarySubSector); break; - case "Donor Agency": + case ImporterConstants.DONOR_AGENCY: logger.info("Getting donor"); - updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), donorAgencyCode, session, "donor"); + updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), donorAgencyCode, session, ImporterConstants.ORG_TYPE_DONOR); break; - case "Responsible Organization": - responsibleOrgId = updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), responsibleOrgCode, session, "responsibleOrg"); + case ImporterConstants.RESPONSIBLE_ORGANIZATION: + responsibleOrgId = updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), responsibleOrgCode, session, ImporterConstants.ORG_TYPE_RESPONSIBLE_ORG); break; - case "Beneficiary Agency": - responsibleOrgId = updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), responsibleOrgCode, session, "beneficiaryAgency"); + case ImporterConstants.BENEFICIARY_AGENCY: + responsibleOrgId = updateOrgs(importDataModel, Objects.requireNonNull(getStringValueFromCell(cell, false)).trim(), responsibleOrgCode, session, ImporterConstants.ORG_TYPE_BENEFICIARY_AGENCY); break; - case "Funding Item": - setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, true, true, false,"Actual", fundingItem, existing); + case ImporterConstants.TRANSACTION_AMOUNT: { + boolean commitment = true, disbursement = true, expenditure = false; + String adjustmentType = ImporterConstants.ADJUSTMENT_TYPE_ACTUAL; + if (config.containsValue(ImporterConstants.MEASURE_TYPE)) { + String measureTypeStr = ImporterUtil.getCellValueByConfig(row, sheet, config, ImporterConstants.MEASURE_TYPE); + ImporterUtil.MeasureTypeResult parsed = parseMeasureType(measureTypeStr); + if (parsed != null) { + commitment = parsed.commitment; + disbursement = parsed.disbursement; + expenditure = parsed.expenditure; + adjustmentType = parsed.adjustmentType; + } + } + setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, commitment, disbursement, expenditure, adjustmentType, fundingItem, existing); break; - case "Planned Commitment": - setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, true, false,false, "Planned", fundingItem, existing); + } + case ImporterConstants.PLANNED_COMMITMENT: + setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, true, false,false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED, fundingItem, existing); break; - case "Planned Disbursement": - setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, false, true, false,"Planned", fundingItem, existing); + case ImporterConstants.PLANNED_DISBURSEMENT: + setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, false, true, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED, fundingItem, existing); break; - case "Planned Expenditure": - setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, false, false,true, "Planned", fundingItem, existing); + case ImporterConstants.PLANNED_EXPENDITURE: + setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, false, false,true, ImporterConstants.ADJUSTMENT_TYPE_PLANNED, fundingItem, existing); break; - case "Actual Commitment": - setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, true, false, false,"Actual", fundingItem, existing); + case ImporterConstants.ACTUAL_COMMITMENT: + setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, true, false, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL, fundingItem, existing); break; - case "Actual Disbursement": - setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, false, true, false,"Actual", fundingItem, existing); + case ImporterConstants.ACTUAL_DISBURSEMENT: + setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, false, true, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL, fundingItem, existing); break; - case "Actual Expenditure": - setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, false, false,true, "Actual", fundingItem, existing); + case ImporterConstants.ACTUAL_EXPENDITURE: + setAFundingItemForExcel(sheet, config, row, entry, importDataModel, session, cell, false, false,true, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL, fundingItem, existing); break; - case "Reporting Date": + case ImporterConstants.MEASURE_TYPE: + break; + case ImporterConstants.REPORTING_DATE: default: logger.error("Unexpected value: " + entry.getValue()); break; @@ -280,7 +298,7 @@ public static void processBatch(List batch,Sheet sheet, HttpServletRequest Long activityId = importTheData(importDataModel, session, importedProject, componentName, componentCode, responsibleOrgId, fundings, existing); - if (activityId != null && config.containsValue("Indicator Name")) { + if (activityId != null && config.containsValue(ImporterConstants.INDICATOR_NAME)) { try { addIndicatorDataToActivity(activityId, row, sheet, config, session); } catch (Exception e) { diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java index 7a3dfcd8c08..9ef0e01cdd5 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java @@ -12,7 +12,9 @@ import org.digijava.module.aim.action.dataimporter.dbentity.ImportedFilesRecord; import org.digijava.module.aim.action.dataimporter.dbentity.ImportedProject; import org.digijava.module.aim.action.dataimporter.model.Funding; +import org.digijava.module.aim.action.dataimporter.util.ImporterConstants; import org.digijava.module.aim.action.dataimporter.model.ImportDataModel; +import org.digijava.module.aim.action.dataimporter.util.ImporterUtil; import org.digijava.module.aim.dbentity.AmpActivityVersion; import org.digijava.module.aim.util.FeaturesUtil; import org.digijava.module.aim.util.TeamMemberUtil; @@ -97,14 +99,14 @@ private static void processBatch(List> batch, HttpServletRe OffsetDateTime now = OffsetDateTime.now(ZoneOffset.UTC); importDataModel.setCreation_date(now.format(formatter)); setStatus(importDataModel); - String componentName= row.get(getKey(config, "Component Name")); - String componentCode= row.get(getKey(config, "Component Code")); - String projectCode= row.get(getKey(config, "Project Code")); - String projectTitle= row.get(getKey(config, "Project Title")); - String projectDesc= row.get(getKey(config, "Project Description")); - String objective= row.get(getKey(config, "Objective")); - String primarySubSector= row.get(getKey(config, "Primary Subsector")); - String secondarySubSector= row.get(getKey(config, "Secondary Subsector")); + String componentName= row.get(getKey(config, ImporterConstants.COMPONENT_NAME)); + String componentCode= row.get(getKey(config, ImporterConstants.COMPONENT_CODE)); + String projectCode= row.get(getKey(config, ImporterConstants.PROJECT_CODE)); + String projectTitle= row.get(getKey(config, ImporterConstants.PROJECT_TITLE)); + String projectDesc= row.get(getKey(config, ImporterConstants.PROJECT_DESCRIPTION)); + String objective= row.get(getKey(config, ImporterConstants.OBJECTIVE)); + String primarySubSector= row.get(getKey(config, ImporterConstants.PRIMARY_SUBSECTOR)); + String secondarySubSector= row.get(getKey(config, ImporterConstants.SECONDARY_SUBSECTOR)); AmpActivityVersion existing = existingActivity(projectTitle,projectCode,session); if (existing!=null && SKIP_EXISTING) { @@ -118,52 +120,67 @@ private static void processBatch(List> batch, HttpServletRe importDataModel.setProject_code(projectCode); importDataModel.setDescription(projectDesc); - String donorAgencyCode= row.get(getKey(config, "Donor Agency Code")); - String responsibleOrgCode= row.get(getKey(config, "Responsible Organization Code")); + String donorAgencyCode= row.get(getKey(config, ImporterConstants.DONOR_AGENCY_CODE)); + String responsibleOrgCode= row.get(getKey(config, ImporterConstants.RESPONSIBLE_ORGANIZATION_CODE)); Long responsibleOrgId=null; logger.info("Configuration: "+config); for (Map.Entry entry : config.entrySet()) { Funding fundingItem = new Funding(); switch (entry.getValue()) { - case "Project Location": + case ImporterConstants.PROJECT_LOCATION: updateLocations(importDataModel, row.get(entry.getKey().trim()),session); break; - case "Primary Sector": + case ImporterConstants.PRIMARY_SECTOR: updateSectors(importDataModel, row.get(entry.getKey().trim()), session, true, primarySubSector); break; - case "Secondary Sector": + case ImporterConstants.SECONDARY_SECTOR: updateSectors(importDataModel, row.get(entry.getKey().trim()), session, false, secondarySubSector); break; - case "Donor Agency": - updateOrgs(importDataModel,row.get(entry.getKey().trim()),donorAgencyCode, session, "donor"); + case ImporterConstants.DONOR_AGENCY: + updateOrgs(importDataModel,row.get(entry.getKey().trim()),donorAgencyCode, session, ImporterConstants.ORG_TYPE_DONOR); break; - case "Responsible Organization": - responsibleOrgId=updateOrgs(importDataModel,row.get(entry.getKey().trim()),responsibleOrgCode, session, "responsibleOrg"); + case ImporterConstants.RESPONSIBLE_ORGANIZATION: + responsibleOrgId=updateOrgs(importDataModel,row.get(entry.getKey().trim()),responsibleOrgCode, session, ImporterConstants.ORG_TYPE_RESPONSIBLE_ORG); break; - case "Beneficiary Agency": - responsibleOrgId=updateOrgs(importDataModel,row.get(entry.getKey().trim()),responsibleOrgCode, session, "beneficiaryAgency"); + case ImporterConstants.BENEFICIARY_AGENCY: + responsibleOrgId=updateOrgs(importDataModel,row.get(entry.getKey().trim()),responsibleOrgCode, session, ImporterConstants.ORG_TYPE_BENEFICIARY_AGENCY); break; - case "Funding Item": - setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), true, true, false,"Actual", fundingItem, existing); + case ImporterConstants.TRANSACTION_AMOUNT: { + boolean commitment = true, disbursement = true, expenditure = false; + String adjustmentType = ImporterConstants.ADJUSTMENT_TYPE_ACTUAL; + if (config.containsValue(ImporterConstants.MEASURE_TYPE)) { + String measureTypeStr = row.get(getKey(config, ImporterConstants.MEASURE_TYPE)); + ImporterUtil.MeasureTypeResult parsed = parseMeasureType(measureTypeStr); + if (parsed != null) { + commitment = parsed.commitment; + disbursement = parsed.disbursement; + expenditure = parsed.expenditure; + adjustmentType = parsed.adjustmentType; + } + } + setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), commitment, disbursement, expenditure, adjustmentType, fundingItem, existing); break; - case "Planned Commitment": - setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), true, false, false,"Planned", fundingItem, existing); + } + case ImporterConstants.PLANNED_COMMITMENT: + setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), true, false, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED, fundingItem, existing); break; - case "Planned Disbursement": - setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), false, true, false,"Planned", fundingItem, existing); + case ImporterConstants.PLANNED_DISBURSEMENT: + setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), false, true, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED, fundingItem, existing); break; - case "Planned Expenditure": - setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), false, false,true, "Planned", fundingItem, existing); + case ImporterConstants.PLANNED_EXPENDITURE: + setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), false, false,true, ImporterConstants.ADJUSTMENT_TYPE_PLANNED, fundingItem, existing); break; - case "Actual Commitment": - setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), true, false, false,"Actual", fundingItem, existing); + case ImporterConstants.ACTUAL_COMMITMENT: + setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), true, false, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL, fundingItem, existing); break; - case "Actual Disbursement": - setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), false, true, false,"Actual", fundingItem, existing); + case ImporterConstants.ACTUAL_DISBURSEMENT: + setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), false, true, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL, fundingItem, existing); break; - case "Actual Expenditure": - setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), false, false,true, "Actual", fundingItem, existing); + case ImporterConstants.ACTUAL_EXPENDITURE: + setAFundingItemForTxt(config, row, entry, importDataModel, session, Double.parseDouble(row.get(entry.getKey().trim())), false, false,true, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL, fundingItem, existing); + break; + case ImporterConstants.MEASURE_TYPE: break; default: logger.error("Unexpected value: " + entry.getValue()); diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterConstants.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterConstants.java new file mode 100644 index 00000000000..88844cd5650 --- /dev/null +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterConstants.java @@ -0,0 +1,74 @@ +package org.digijava.module.aim.action.dataimporter.util; + +/** + * Central place for all string constants used by the Data Importer (Excel, Txt, config). + * These are the field names used in column-to-entity mapping and in switch/case logic. + */ +public final class ImporterConstants { + + private ImporterConstants() { + } + + // ----- Adjustment types (funding: Actual vs Planned) ----- + public static final String ADJUSTMENT_TYPE_ACTUAL = "Actual"; + public static final String ADJUSTMENT_TYPE_PLANNED = "Planned"; + + // ----- Organization role types (for updateOrgs) ----- + public static final String ORG_TYPE_DONOR = "donor"; + public static final String ORG_TYPE_RESPONSIBLE_ORG = "responsibleOrg"; + public static final String ORG_TYPE_BENEFICIARY_AGENCY = "beneficiaryAgency"; + + // ----- Entity / column field names (template mapping) ----- + public static final String PROJECT_TITLE = "Project Title"; + public static final String PROJECT_CODE = "Project Code"; + public static final String OBJECTIVE = "Objective"; + public static final String PROJECT_DESCRIPTION = "Project Description"; + public static final String PRIMARY_SECTOR = "Primary Sector"; + public static final String SECONDARY_SECTOR = "Secondary Sector"; + public static final String PROJECT_LOCATION = "Project Location"; + public static final String PROJECT_START_DATE = "Project Start Date"; + public static final String PROJECT_END_DATE = "Project End Date"; + public static final String DONOR_AGENCY = "Donor Agency"; + public static final String DONOR_AGENCY_CODE = "Donor Agency Code"; + public static final String EXCHANGE_RATE = "Exchange Rate"; + public static final String RESPONSIBLE_ORGANIZATION = "Responsible Organization"; + public static final String RESPONSIBLE_ORGANIZATION_CODE = "Responsible Organization Code"; + public static final String EXECUTING_AGENCY = "Executing Agency"; + public static final String IMPLEMENTING_AGENCY = "Implementing Agency"; + public static final String BENEFICIARY_AGENCY = "Beneficiary Agency"; + + public static final String ACTUAL_DISBURSEMENT = "Actual Disbursement"; + public static final String ACTUAL_COMMITMENT = "Actual Commitment"; + public static final String ACTUAL_EXPENDITURE = "Actual Expenditure"; + public static final String PLANNED_DISBURSEMENT = "Planned Disbursement"; + public static final String PLANNED_COMMITMENT = "Planned Commitment"; + public static final String PLANNED_EXPENDITURE = "Planned Expenditure"; + public static final String TRANSACTION_AMOUNT = "Transaction Amount"; + public static final String MEASURE_TYPE = "Measure Type"; + public static final String TRANSACTION_DATE = "Transaction Date"; + public static final String FINANCING_INSTRUMENT = "Financing Instrument"; + public static final String TYPE_OF_ASSISTANCE = "Type Of Assistance"; + public static final String PRIMARY_SUBSECTOR = "Primary Subsector"; + public static final String SECONDARY_SUBSECTOR = "Secondary Subsector"; + public static final String CURRENCY = "Currency"; + public static final String COMPONENT_NAME = "Component Name"; + public static final String COMPONENT_CODE = "Component Code"; + + public static final String REPORTING_DATE = "Reporting Date"; + + // ----- Indicator (M&E) columns ----- + public static final String INDICATOR_NAME = "Indicator Name"; + public static final String PROGRAM_NAME = "Program Name"; + public static final String LOCATION = "Location"; + public static final String ORIGINAL_BASE_VALUE = "Original Base Value"; + public static final String ORIGINAL_BASE_VALUE_DATE = "Original Base Value Date"; + public static final String REVISED_BASE_VALUE = "Revised Base Value"; + public static final String REVISED_BASE_VALUE_DATE = "Revised Base Value Date"; + public static final String ORIGINAL_TARGET_VALUE = "Original Target Value"; + public static final String ORIGINAL_TARGET_VALUE_DATE = "Original Target Value Date"; + public static final String REVISED_TARGET_VALUE = "Revised Target Value"; + public static final String REVISED_TARGET_VALUE_DATE = "Revised Target Value Date"; + public static final String ACTUAL_VALUE = "Actual Value"; + public static final String ACTUAL_VALUE_DATE = "Actual Value Date"; + public static final String UNIT_OF_MEASURE = "Unit of Measure"; +} diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java index 73d9ea26f62..93953bf23f9 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java @@ -38,8 +38,6 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; -import java.math.BigDecimal; -import java.math.RoundingMode; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; @@ -74,19 +72,19 @@ private static Double parseDouble(String number) { public static Funding setAFundingItemForExcel(Sheet sheet, Map config, Row row, Map.Entry entry, ImportDataModel importDataModel, Session session, Cell cell, boolean commitment, boolean disbursement, boolean expenditure, String adjustmentType, Funding fundingItem, AmpActivityVersion existingActivity) { - int detailColumn = getColumnIndexByName(sheet, getKey(config, "Financing Instrument")); + int detailColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.FINANCING_INSTRUMENT)); String finInstrument = detailColumn >= 0 ? getStringValueFromCell(row.getCell(detailColumn), false) : ""; - detailColumn = getColumnIndexByName(sheet, getKey(config, "Exchange Rate")); + detailColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.EXCHANGE_RATE)); String exchangeRate = detailColumn >= 0 ? getStringValueFromCell(row.getCell(detailColumn), false) : ""; Double exchangeRateValue = !exchangeRate.isEmpty() ? parseDouble(exchangeRate) : Double.valueOf(0.0); - detailColumn = getColumnIndexByName(sheet, getKey(config, "Type Of Assistance")); + detailColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.TYPE_OF_ASSISTANCE)); String typeOfAss = detailColumn >= 0 ? getStringValueFromCell(row.getCell(detailColumn), false) : ""; - int separateFundingDateColumn = getColumnIndexByName(sheet, getKey(config, "Transaction Date")); + int separateFundingDateColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.TRANSACTION_DATE)); String separateFundingDate = separateFundingDateColumn >= 0 ? getDateFromExcel(row, separateFundingDateColumn) : null; - int currencyCodeColumn = getColumnIndexByName(sheet, getKey(config, "Currency")); + int currencyCodeColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.CURRENCY)); String currencyCode = currencyCodeColumn >= 0 ? getStringValueFromCell(row.getCell(currencyCodeColumn), true) : CurrencyUtil.getDefaultCurrency().getCurrencyCode(); if (existingActivity != null) { String existingActivityCurrencyCode = getCurrencyCodeFromExistingImported(existingActivity.getName()); @@ -96,15 +94,15 @@ public static Funding setAFundingItemForExcel(Sheet sheet, Map c } saveCurrencyCode(currencyCode, importDataModel.getProject_title()); Funding funding; - int componentNameColumn = getColumnIndexByName(sheet, getKey(config, "Component Name")); + int componentNameColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.COMPONENT_NAME)); String componentName = componentNameColumn >= 0 ? getStringValueFromCell(row.getCell(componentNameColumn), true) : null; if (importDataModel.getDonor_organization() == null || importDataModel.getDonor_organization().isEmpty()) { - if (!config.containsValue("Donor Agency")) { + if (!config.containsValue(ImporterConstants.DONOR_AGENCY)) { funding = updateFunding(fundingItem, importDataModel, getNumericValueFromCell(cell), entry.getKey(), separateFundingDate, getRandomOrg(session), typeOfAss, finInstrument, commitment, disbursement, expenditure, adjustmentType, currencyCode, componentName, exchangeRateValue); } else { - int columnIndex1 = getColumnIndexByName(sheet, getKey(config, "Donor Agency")); - int donorAgencyCodeColumn = getColumnIndexByName(sheet, getKey(config, "Donor Agency Code")); + int columnIndex1 = getColumnIndexByName(sheet, getKey(config, ImporterConstants.DONOR_AGENCY)); + int donorAgencyCodeColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.DONOR_AGENCY_CODE)); String donorAgencyCode = donorAgencyCodeColumn >= 0 ? getStringValueFromCell(row.getCell(donorAgencyCodeColumn), true) : null; updateOrgs(importDataModel, columnIndex1 >= 0 ? Objects.requireNonNull(getStringValueFromCell(row.getCell(columnIndex1), false)).trim() : "no org", donorAgencyCode, session, "donor"); funding = updateFunding(fundingItem, importDataModel, getNumericValueFromCell(cell), entry.getKey(), separateFundingDate, new ArrayList<>(importDataModel.getDonor_organization()).get(0).getOrganization(), typeOfAss, finInstrument, commitment, disbursement, expenditure, adjustmentType, currencyCode, componentName, exchangeRateValue); @@ -119,17 +117,17 @@ public static Funding setAFundingItemForExcel(Sheet sheet, Map c public static Funding setAFundingItemForTxt(Map row, Map config, Map.Entry entry, ImportDataModel importDataModel, Session session, Number value, boolean commitment, boolean disbursement, boolean expenditure, String adjustmentType, Funding fundingItem, AmpActivityVersion existingActivity) { - String finInstrument = row.get(getKey(config, "Financing Instrument")); + String finInstrument = row.get(getKey(config, ImporterConstants.FINANCING_INSTRUMENT)); finInstrument = finInstrument != null ? finInstrument : ""; - String typeOfAss = row.get(getKey(config, "Type Of Assistance")); + String typeOfAss = row.get(getKey(config, ImporterConstants.TYPE_OF_ASSISTANCE)); typeOfAss = typeOfAss != null ? typeOfAss : ""; Funding funding; - String separateFundingDate = row.get(getKey(config, "Transaction Date")); + String separateFundingDate = row.get(getKey(config, ImporterConstants.TRANSACTION_DATE)); separateFundingDate = separateFundingDate != null ? separateFundingDate : ""; - String currencyCode = row.get(getKey(config, "Currency")); + String currencyCode = row.get(getKey(config, ImporterConstants.CURRENCY)); currencyCode = currencyCode != null ? currencyCode : CurrencyUtil.getDefaultCurrency().getCurrencyCode(); if (existingActivity != null) { String existingActivityCurrencyCode = getCurrencyCodeFromExistingImported(existingActivity.getName()); @@ -138,23 +136,23 @@ public static Funding setAFundingItemForTxt(Map row, Map(importDataModel.getDonor_organization()).get(0).getOrganization(), typeOfAss, finInstrument, commitment, disbursement, expenditure, adjustmentType, currencyCode, componentName, exchangeRateValue); @@ -392,6 +390,58 @@ public static K getKey(Map map, V value) { return null; } + /** + * Result of parsing a Measure Type string (e.g. "PC - Planned Commitment"). + * Used to set commitment/disbursement/expenditure and Actual/Planned for funding. + */ + public static class MeasureTypeResult { + public final boolean commitment; + public final boolean disbursement; + public final boolean expenditure; + public final String adjustmentType; // "Actual" or "Planned" + + public MeasureTypeResult(boolean commitment, boolean disbursement, boolean expenditure, String adjustmentType) { + this.commitment = commitment; + this.disbursement = disbursement; + this.expenditure = expenditure; + this.adjustmentType = adjustmentType; + } + } + + /** + * Parses a Measure Type value from the template (e.g. "PC - Planned Commitment", "AC", or "Actual Commitment"). + * @return MeasureTypeResult or null if not recognized + */ + public static MeasureTypeResult parseMeasureType(String value) { + if (value == null) return null; + String s = value.trim(); + if (s.isEmpty()) return null; + // AC / PC / AD / PD / AE / PE + if (s.equalsIgnoreCase("AC")) return new MeasureTypeResult(true, false, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (s.equalsIgnoreCase("PC")) return new MeasureTypeResult(true, false, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + if (s.equalsIgnoreCase("AD")) return new MeasureTypeResult(false, true, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (s.equalsIgnoreCase("PD")) return new MeasureTypeResult(false, true, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + if (s.equalsIgnoreCase("AE")) return new MeasureTypeResult(false, false, true, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (s.equalsIgnoreCase("PE")) return new MeasureTypeResult(false, false, true, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + // Full form "PC - Planned Commitment" or label only "Planned Commitment" + if (s.contains(" - ")) { + String code = s.substring(0, s.indexOf(" - ")).trim(); + if (code.equalsIgnoreCase("AC")) return new MeasureTypeResult(true, false, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (code.equalsIgnoreCase("PC")) return new MeasureTypeResult(true, false, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + if (code.equalsIgnoreCase("AD")) return new MeasureTypeResult(false, true, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (code.equalsIgnoreCase("PD")) return new MeasureTypeResult(false, true, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + if (code.equalsIgnoreCase("AE")) return new MeasureTypeResult(false, false, true, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (code.equalsIgnoreCase("PE")) return new MeasureTypeResult(false, false, true, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + } + if (s.equalsIgnoreCase(ImporterConstants.ACTUAL_COMMITMENT)) return new MeasureTypeResult(true, false, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (s.equalsIgnoreCase(ImporterConstants.PLANNED_COMMITMENT)) return new MeasureTypeResult(true, false, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + if (s.equalsIgnoreCase(ImporterConstants.ACTUAL_DISBURSEMENT)) return new MeasureTypeResult(false, true, false, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (s.equalsIgnoreCase(ImporterConstants.PLANNED_DISBURSEMENT)) return new MeasureTypeResult(false, true, false, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + if (s.equalsIgnoreCase(ImporterConstants.ACTUAL_EXPENDITURE)) return new MeasureTypeResult(false, false, true, ImporterConstants.ADJUSTMENT_TYPE_ACTUAL); + if (s.equalsIgnoreCase(ImporterConstants.PLANNED_EXPENDITURE)) return new MeasureTypeResult(false, false, true, ImporterConstants.ADJUSTMENT_TYPE_PLANNED); + return null; + } + public static String findYearSubstring(String text) { Pattern pattern = Pattern.compile("(?:19|20)\\d{2}"); Matcher matcher = pattern.matcher(text); @@ -1263,17 +1313,17 @@ public static Date parseDateDefaultToday(Row row, Sheet sheet, Map config, Session session) { if (activityId == null || config == null || row == null || sheet == null) return; - if (getKey(config, "Indicator Name") == null || getKey(config, "Location") == null || getKey(config, "Actual Value") == null) { + if (getKey(config, ImporterConstants.INDICATOR_NAME) == null || getKey(config, ImporterConstants.LOCATION) == null || getKey(config, ImporterConstants.ACTUAL_VALUE) == null) { return; } - String indicatorName = getCellValueByConfig(row, sheet, config, "Indicator Name"); - String locationName = getCellValueByConfig(row, sheet, config, "Location"); + String indicatorName = getCellValueByConfig(row, sheet, config, ImporterConstants.INDICATOR_NAME); + String locationName = getCellValueByConfig(row, sheet, config, ImporterConstants.LOCATION); if (indicatorName == null || indicatorName.trim().isEmpty() || locationName == null || locationName.trim().isEmpty()) { return; } indicatorName = indicatorName.trim(); locationName = locationName.trim(); - String programName = getCellValueByConfig(row, sheet, config, "Program Name"); + String programName = getCellValueByConfig(row, sheet, config, ImporterConstants.PROGRAM_NAME); if (programName != null) programName = programName.trim(); AmpTheme programTheme = null; @@ -1334,20 +1384,20 @@ public static void addIndicatorDataToActivity(Long activityId, Row row, Sheet sh AmpIndicatorGlobalValue existingBase = indicator.getBaseValue(); - double origBase = parseDoubleFromConfig(row, sheet, config, "Original Base Value"); - boolean hasOrigBase = getKey(config, "Original Base Value") != null && !Double.isNaN(origBase); - double revBase = parseDoubleFromConfig(row, sheet, config, "Revised Base Value"); - boolean hasRevBase = getKey(config, "Revised Base Value") != null && !Double.isNaN(revBase); - double origTarget = parseDoubleFromConfig(row, sheet, config, "Original Target Value"); - double revTarget = parseDoubleFromConfig(row, sheet, config, "Revised Target Value"); - double actualVal = parseDoubleFromConfig(row, sheet, config, "Actual Value"); + double origBase = parseDoubleFromConfig(row, sheet, config, ImporterConstants.ORIGINAL_BASE_VALUE); + boolean hasOrigBase = getKey(config, ImporterConstants.ORIGINAL_BASE_VALUE) != null && !Double.isNaN(origBase); + double revBase = parseDoubleFromConfig(row, sheet, config, ImporterConstants.REVISED_BASE_VALUE); + boolean hasRevBase = getKey(config, ImporterConstants.REVISED_BASE_VALUE) != null && !Double.isNaN(revBase); + double origTarget = parseDoubleFromConfig(row, sheet, config, ImporterConstants.ORIGINAL_TARGET_VALUE); + double revTarget = parseDoubleFromConfig(row, sheet, config, ImporterConstants.REVISED_TARGET_VALUE); + double actualVal = parseDoubleFromConfig(row, sheet, config, ImporterConstants.ACTUAL_VALUE); if (Double.isNaN(actualVal)) actualVal = 0.0; - Date origBaseDate = parseDateDefaultToday(row, sheet, config, "Original Base Value Date"); - Date revBaseDate = parseDateDefaultToday(row, sheet, config, "Revised Base Value Date"); - Date origTargetDate = parseDateDefaultToday(row, sheet, config, "Original Target Value Date"); - Date revTargetDate = parseDateDefaultToday(row, sheet, config, "Revised Target Value Date"); - Date actualDate = parseDateDefaultToday(row, sheet, config, "Actual Value Date"); + Date origBaseDate = parseDateDefaultToday(row, sheet, config, ImporterConstants.ORIGINAL_BASE_VALUE_DATE); + Date revBaseDate = parseDateDefaultToday(row, sheet, config, ImporterConstants.REVISED_BASE_VALUE_DATE); + Date origTargetDate = parseDateDefaultToday(row, sheet, config, ImporterConstants.ORIGINAL_TARGET_VALUE_DATE); + Date revTargetDate = parseDateDefaultToday(row, sheet, config, ImporterConstants.REVISED_TARGET_VALUE_DATE); + Date actualDate = parseDateDefaultToday(row, sheet, config, ImporterConstants.ACTUAL_VALUE_DATE); double baseOrigVal = hasOrigBase ? origBase : (existingBase != null && existingBase.getOriginalValue() != null ? existingBase.getOriginalValue() : 0.0); double baseRevVal = hasRevBase ? revBase : (existingBase != null && existingBase.getRevisedValue() != null ? existingBase.getRevisedValue() : 0.0); @@ -1355,7 +1405,7 @@ public static void addIndicatorDataToActivity(Long activityId, Row row, Sheet sh double targetRevVal = Double.isNaN(revTarget) ? 0.0 : revTarget; Set values = new HashSet<>(); - if (getKey(config, "Original Base Value") != null || getKey(config, "Revised Base Value") != null || existingBase != null) { + if (getKey(config, ImporterConstants.ORIGINAL_BASE_VALUE) != null || getKey(config, ImporterConstants.REVISED_BASE_VALUE) != null || existingBase != null) { AmpIndicatorValue baseOrig = new AmpIndicatorValue(AmpIndicatorValue.BASE); baseOrig.setValue(baseOrigVal); baseOrig.setValueDate(origBaseDate); @@ -1367,7 +1417,7 @@ public static void addIndicatorDataToActivity(Long activityId, Row row, Sheet sh baseRev.setIndicatorConnection(ia); values.add(baseRev); } - if (getKey(config, "Original Target Value") != null || getKey(config, "Revised Target Value") != null) { + if (getKey(config, ImporterConstants.ORIGINAL_TARGET_VALUE) != null || getKey(config, ImporterConstants.REVISED_TARGET_VALUE) != null) { AmpIndicatorValue tOrig = new AmpIndicatorValue(AmpIndicatorValue.TARGET); tOrig.setValue(targetOrigVal); tOrig.setValueDate(origTargetDate); @@ -1383,7 +1433,7 @@ public static void addIndicatorDataToActivity(Long activityId, Row row, Sheet sh actual.setValue(actualVal); actual.setValueDate(actualDate); actual.setIndicatorConnection(ia); - String unit = getCellValueByConfig(row, sheet, config, "Unit of Measure"); + String unit = getCellValueByConfig(row, sheet, config, ImporterConstants.UNIT_OF_MEASURE); if (unit != null && !unit.trim().isEmpty()) actual.setComment("Unit: " + unit.trim()); values.add(actual); @@ -1465,7 +1515,7 @@ public static AmpTheme getOrCreateProgramByName(String programName, Session sess } } - private static String getCellValueByConfig(Row row, Sheet sheet, Map config, String fieldName) { + public static String getCellValueByConfig(Row row, Sheet sheet, Map config, String fieldName) { String key = getKey(config, fieldName); if (key == null) return null; int col = getColumnIndexByName(sheet, key); From d308833289470ac9c7dc73233b6ac04d5064b10c Mon Sep 17 00:00:00 2001 From: brianbrix Date: Sun, 15 Feb 2026 19:56:49 +0300 Subject: [PATCH 06/59] Activity status getOrCreate --- .../aim/action/dataimporter/DataImporter.java | 1 + .../action/dataimporter/ExcelImporter.java | 15 +++- .../action/dataimporter/TxtDataImporter.java | 12 +++- .../dataimporter/util/ImporterConstants.java | 1 + .../dataimporter/util/ImporterUtil.java | 70 ++++++++++++++++++- 5 files changed, 94 insertions(+), 5 deletions(-) diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java index 9b2fd297019..299785d67f0 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java @@ -497,6 +497,7 @@ private List getEntityFieldsInfo() { fieldsInfos.add(ImporterConstants.COMPONENT_NAME); fieldsInfos.add(ImporterConstants.COMPONENT_CODE); fieldsInfos.add(ImporterConstants.BENEFICIARY_AGENCY); + fieldsInfos.add(ImporterConstants.PROJECT_STATUS); // Indicator columns for M&E import fieldsInfos.add(ImporterConstants.INDICATOR_NAME); fieldsInfos.add(ImporterConstants.PROGRAM_NAME); diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java index 294b8557c29..7932735c045 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java @@ -173,7 +173,6 @@ public static void processBatch(List batch,Sheet sheet, HttpServletRequest importDataModel.setIs_draft(true); OffsetDateTime now = OffsetDateTime.now(ZoneOffset.UTC); importDataModel.setCreation_date(now.format(formatter)); - setStatus(importDataModel); int componentCodeColumn = getColumnIndexByName(sheet, getKey(config, ImporterConstants.COMPONENT_CODE)); String componentCode = componentCodeColumn >= 0 ? getStringValueFromCell(row.getCell(componentCodeColumn),true) : null; @@ -208,6 +207,17 @@ public static void processBatch(List batch,Sheet sheet, HttpServletRequest String projectDesc = projectDescColumn >= 0 ? getStringValueFromCell(row.getCell(projectDescColumn),false) : null; importDataModel.setDescription(projectDesc); + if (config.containsValue(ImporterConstants.PROJECT_STATUS)) { + String projectStatusStr = ImporterUtil.getCellValueByConfig(row, sheet, config, ImporterConstants.PROJECT_STATUS); + if (projectStatusStr != null && !projectStatusStr.trim().isEmpty()) { + Long statusId = ImporterUtil.getOrCreateActivityStatusCategoryValue(projectStatusStr.trim(), session); + if (statusId != null) { + importDataModel.setActivity_status(statusId); + } + } + } + setStatus(importDataModel); + AmpActivityVersion existing = existingActivity(projectTitle, projectCode, session); Long responsibleOrgId = null; @@ -282,6 +292,8 @@ public static void processBatch(List batch,Sheet sheet, HttpServletRequest break; case ImporterConstants.MEASURE_TYPE: break; + case ImporterConstants.PROJECT_STATUS: + break; case ImporterConstants.REPORTING_DATE: default: logger.error("Unexpected value: " + entry.getValue()); @@ -296,7 +308,6 @@ public static void processBatch(List batch,Sheet sheet, HttpServletRequest } - Long activityId = importTheData(importDataModel, session, importedProject, componentName, componentCode, responsibleOrgId, fundings, existing); if (activityId != null && config.containsValue(ImporterConstants.INDICATOR_NAME)) { try { diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java index 9ef0e01cdd5..0453516a45c 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java @@ -98,7 +98,6 @@ private static void processBatch(List> batch, HttpServletRe importDataModel.setIs_draft(true); OffsetDateTime now = OffsetDateTime.now(ZoneOffset.UTC); importDataModel.setCreation_date(now.format(formatter)); - setStatus(importDataModel); String componentName= row.get(getKey(config, ImporterConstants.COMPONENT_NAME)); String componentCode= row.get(getKey(config, ImporterConstants.COMPONENT_CODE)); String projectCode= row.get(getKey(config, ImporterConstants.PROJECT_CODE)); @@ -107,6 +106,7 @@ private static void processBatch(List> batch, HttpServletRe String objective= row.get(getKey(config, ImporterConstants.OBJECTIVE)); String primarySubSector= row.get(getKey(config, ImporterConstants.PRIMARY_SUBSECTOR)); String secondarySubSector= row.get(getKey(config, ImporterConstants.SECONDARY_SUBSECTOR)); + String projectStatusStr = row.get(getKey(config, ImporterConstants.PROJECT_STATUS)); AmpActivityVersion existing = existingActivity(projectTitle,projectCode,session); if (existing!=null && SKIP_EXISTING) { @@ -120,6 +120,14 @@ private static void processBatch(List> batch, HttpServletRe importDataModel.setProject_code(projectCode); importDataModel.setDescription(projectDesc); + if (projectStatusStr != null && !projectStatusStr.trim().isEmpty()) { + Long statusId = getOrCreateActivityStatusCategoryValue(projectStatusStr.trim(), session); + if (statusId != null) { + importDataModel.setActivity_status(statusId); + } + } + setStatus(importDataModel); + String donorAgencyCode= row.get(getKey(config, ImporterConstants.DONOR_AGENCY_CODE)); String responsibleOrgCode= row.get(getKey(config, ImporterConstants.RESPONSIBLE_ORGANIZATION_CODE)); Long responsibleOrgId=null; @@ -182,6 +190,8 @@ private static void processBatch(List> batch, HttpServletRe break; case ImporterConstants.MEASURE_TYPE: break; + case ImporterConstants.PROJECT_STATUS: + break; default: logger.error("Unexpected value: " + entry.getValue()); break; diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterConstants.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterConstants.java index 88844cd5650..6e86622c2e3 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterConstants.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterConstants.java @@ -55,6 +55,7 @@ private ImporterConstants() { public static final String COMPONENT_CODE = "Component Code"; public static final String REPORTING_DATE = "Reporting Date"; + public static final String PROJECT_STATUS = "Project Status"; // ----- Indicator (M&E) columns ----- public static final String INDICATOR_NAME = "Indicator Name"; diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java index 93953bf23f9..525b500890f 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java @@ -25,8 +25,10 @@ import org.digijava.module.aim.util.CurrencyUtil; import org.digijava.module.aim.util.FeaturesUtil; import org.digijava.module.aim.util.ProgramUtil; +import org.digijava.module.categorymanager.dbentity.AmpCategoryClass; import org.digijava.module.categorymanager.dbentity.AmpCategoryValue; import org.digijava.module.categorymanager.util.CategoryConstants; +import org.digijava.module.categorymanager.util.CategoryManagerUtil; import org.hibernate.Query; import org.hibernate.Session; import org.hibernate.type.StringType; @@ -618,6 +620,64 @@ private static Long getCategoryValue(String constantKey, String categoryKey, Str return categoryId; } + /** + * Resolves activity (project) status by value: looks up existing category value for ACTIVITY_STATUS_KEY; + * if not found in DB, creates a new category value and returns its id. + * @param statusValue value from the file (e.g. "Ongoing", "Completed") + * @param session current session (used for create and flush) + * @return category value id, or null if statusValue is null/empty + */ + public static Long getOrCreateActivityStatusCategoryValue(String statusValue, Session session) { + if (statusValue == null || statusValue.trim().isEmpty()) return null; + String trimmed = statusValue.trim(); + String cacheKey = "statusId_" + trimmed; + if (ConstantsMap.containsKey(cacheKey)) { + return ConstantsMap.get(cacheKey); + } + if (!session.isOpen()) { + session = PersistenceManager.getRequestDBSession(); + } + String hql = "SELECT s FROM " + AmpCategoryValue.class.getName() + " s JOIN s.ampCategoryClass c WHERE c.keyName = :categoryKey"; + Query query = session.createQuery(hql); + query.setParameter("categoryKey", CategoryConstants.ACTIVITY_STATUS_KEY); + @SuppressWarnings("unchecked") + List values = (List) query.list(); + if (values != null) { + for (AmpCategoryValue cv : values) { + if (cv.getValue() != null && cv.getValue().equalsIgnoreCase(trimmed)) { + Long id = cv.getId(); + ConstantsMap.put(cacheKey, id); + return id; + } + } + } + AmpCategoryClass categoryClass = CategoryManagerUtil.loadAmpCategoryClassByKey(CategoryConstants.ACTIVITY_STATUS_KEY); + if (categoryClass == null) { + logger.warn("Activity status category class not found; cannot create value: " + trimmed); + return null; + } + try { + AmpCategoryValue newValue = new AmpCategoryValue(); + newValue.setValue(trimmed); + newValue.setAmpCategoryClass(categoryClass); + if (categoryClass.getPossibleValues() == null) { + categoryClass.setPossibleValues(new java.util.ArrayList<>()); + } + newValue.setIndex(categoryClass.getPossibleValues().size()); + session.save(newValue); + session.flush(); + Long id = newValue.getId(); + if (id != null) { + ConstantsMap.put(cacheKey, id); + logger.info("Created new activity status category value: " + trimmed + " (id=" + id + ")"); + return id; + } + } catch (Exception e) { + logger.warn("Failed to create activity status value: " + trimmed, e); + } + return null; + } + public static AmpActivityVersion existingActivity(String projectTitle, String projectCode, Session session) { if ((projectTitle == null || projectTitle.trim().isEmpty()) && (projectCode == null || projectCode.trim().isEmpty())) { @@ -649,9 +709,15 @@ public static AmpActivityVersion existingActivity(String projectTitle, String pr return null; } + /** + * Sets default activity status and approval status on the import model. + * If activity_status is already set (e.g. from Project Status column), it is left unchanged. + */ public static void setStatus(ImportDataModel importDataModel) { - Long statusId = getCategoryValue("statusId", CategoryConstants.ACTIVITY_STATUS_KEY, ""); - importDataModel.setActivity_status(statusId); + if (importDataModel.getActivity_status() == null) { + Long statusId = getCategoryValue("statusId", CategoryConstants.ACTIVITY_STATUS_KEY, ""); + importDataModel.setActivity_status(statusId); + } importDataModel.setApproval_status(ApprovalStatus.started.getId()); } From 2e82e7cb553e8e50b78103e1abc1347f5b186734 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Mon, 16 Feb 2026 14:16:27 +0300 Subject: [PATCH 07/59] Use indicator Location column Resolve missing created by --- .../aim/action/dataimporter/DataImporter.java | 1 + .../dataimporter/util/ImporterConstants.java | 3 ++ .../dataimporter/util/ImporterUtil.java | 36 +++++++++++++++++-- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java index 299785d67f0..4eff3282dec 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java @@ -502,6 +502,7 @@ private List getEntityFieldsInfo() { fieldsInfos.add(ImporterConstants.INDICATOR_NAME); fieldsInfos.add(ImporterConstants.PROGRAM_NAME); fieldsInfos.add(ImporterConstants.LOCATION); + fieldsInfos.add(ImporterConstants.INDICATOR_LOCATION); fieldsInfos.add(ImporterConstants.ORIGINAL_BASE_VALUE); fieldsInfos.add(ImporterConstants.ORIGINAL_BASE_VALUE_DATE); fieldsInfos.add(ImporterConstants.REVISED_BASE_VALUE); diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterConstants.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterConstants.java index 6e86622c2e3..8b863b0b0d4 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterConstants.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterConstants.java @@ -60,7 +60,10 @@ private ImporterConstants() { // ----- Indicator (M&E) columns ----- public static final String INDICATOR_NAME = "Indicator Name"; public static final String PROGRAM_NAME = "Program Name"; + /** Used for project-level location (e.g. Project Location). */ public static final String LOCATION = "Location"; + /** Used for matching indicator value to activity location; distinct from project Location. */ + public static final String INDICATOR_LOCATION = "Indicator Location"; public static final String ORIGINAL_BASE_VALUE = "Original Base Value"; public static final String ORIGINAL_BASE_VALUE_DATE = "Original Base Value Date"; public static final String REVISED_BASE_VALUE = "Revised Base Value"; diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java index 525b500890f..7cebe666e1f 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java @@ -25,6 +25,7 @@ import org.digijava.module.aim.util.CurrencyUtil; import org.digijava.module.aim.util.FeaturesUtil; import org.digijava.module.aim.util.ProgramUtil; +import org.digijava.module.aim.util.TeamUtil; import org.digijava.module.categorymanager.dbentity.AmpCategoryClass; import org.digijava.module.categorymanager.dbentity.AmpCategoryValue; import org.digijava.module.categorymanager.util.CategoryConstants; @@ -721,6 +722,32 @@ public static void setStatus(ImportDataModel importDataModel) { importDataModel.setApproval_status(ApprovalStatus.started.getId()); } + private static final String CREATED_BY_KEY = "created_by"; + + /** + * Ensures created_by in the activity map is set to a valid team member id when null, + * so the activity API validator does not reject with "(Invalid field value) created_by". + * For new activities uses current user; for updates uses existing activity's creator. + */ + private static void ensureCreatedBySet(Map map, AmpActivityVersion existing) { + Object createdBy = map.get(CREATED_BY_KEY); + if (createdBy != null) { + return; + } + Long creatorId = null; + if (existing != null && existing.getActivityCreator() != null) { + creatorId = existing.getActivityCreator().getAmpTeamMemId(); + } else { + AmpTeamMember currentMember = TeamUtil.getCurrentAmpTeamMember(); + if (currentMember != null) { + creatorId = currentMember.getAmpTeamMemId(); + } + } + if (creatorId != null) { + map.put(CREATED_BY_KEY, creatorId); + } + } + /** @return activity ID on success, null on skip or failure */ public static Long importTheData(ImportDataModel importDataModel, Session session, ImportedProject importedProject, String componentName, String componentCode, Long responsibleOrgId, List fundings, AmpActivityVersion existing) throws JsonProcessingException { if (!session.isOpen()) { @@ -743,6 +770,7 @@ public static Long importTheData(ImportDataModel importDataModel, Session sessio return null; } if (existing == null) { + ensureCreatedBySet(map, null); logger.info("New activity"); importedProject.setNewProject(true); response = ActivityInterchangeUtils.importActivity(map, false, rules, "activity/new"); @@ -760,6 +788,7 @@ public static Long importTheData(ImportDataModel importDataModel, Session sessio map = objectMapper .convertValue(importDataModel, new TypeReference>() { }); + ensureCreatedBySet(map, existing); response = ActivityInterchangeUtils.importActivity(map, true, rules, "activity/update"); } Long activityId = null; @@ -1379,11 +1408,14 @@ public static Date parseDateDefaultToday(Row row, Sheet sheet, Map config, Session session) { if (activityId == null || config == null || row == null || sheet == null) return; - if (getKey(config, ImporterConstants.INDICATOR_NAME) == null || getKey(config, ImporterConstants.LOCATION) == null || getKey(config, ImporterConstants.ACTUAL_VALUE) == null) { + String locationConfigKey = getKey(config, ImporterConstants.INDICATOR_LOCATION) != null + ? ImporterConstants.INDICATOR_LOCATION + : ImporterConstants.LOCATION; + if (getKey(config, ImporterConstants.INDICATOR_NAME) == null || getKey(config, locationConfigKey) == null || getKey(config, ImporterConstants.ACTUAL_VALUE) == null) { return; } String indicatorName = getCellValueByConfig(row, sheet, config, ImporterConstants.INDICATOR_NAME); - String locationName = getCellValueByConfig(row, sheet, config, ImporterConstants.LOCATION); + String locationName = getCellValueByConfig(row, sheet, config, locationConfigKey); if (indicatorName == null || indicatorName.trim().isEmpty() || locationName == null || locationName.trim().isEmpty()) { return; } From d4bcfa5f2ae62ea54599a068fccc79785f8a6f2d Mon Sep 17 00:00:00 2001 From: brianbrix Date: Mon, 16 Feb 2026 14:31:28 +0300 Subject: [PATCH 08/59] Use indicator Location column Resolve missing created by --- .../dataimporter/util/ImporterUtil.java | 156 +++++++++++++++++- 1 file changed, 148 insertions(+), 8 deletions(-) diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java index 7cebe666e1f..ccf7936f9dd 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java @@ -1475,13 +1475,7 @@ public static void addIndicatorDataToActivity(Long activityId, Row row, Sheet sh } } - IndicatorActivity ia = new IndicatorActivity(); - ia.setActivity(activity); - ia.setIndicator(indicator); - if (activityLocation != null) ia.setActivityLocation(activityLocation); - AmpIndicatorGlobalValue existingBase = indicator.getBaseValue(); - double origBase = parseDoubleFromConfig(row, sheet, config, ImporterConstants.ORIGINAL_BASE_VALUE); boolean hasOrigBase = getKey(config, ImporterConstants.ORIGINAL_BASE_VALUE) != null && !Double.isNaN(origBase); double revBase = parseDoubleFromConfig(row, sheet, config, ImporterConstants.REVISED_BASE_VALUE); @@ -1501,6 +1495,24 @@ public static void addIndicatorDataToActivity(Long activityId, Row row, Sheet sh double baseRevVal = hasRevBase ? revBase : (existingBase != null && existingBase.getRevisedValue() != null ? existingBase.getRevisedValue() : 0.0); double targetOrigVal = Double.isNaN(origTarget) ? 0.0 : origTarget; double targetRevVal = Double.isNaN(revTarget) ? 0.0 : revTarget; + String unit = getCellValueByConfig(row, sheet, config, ImporterConstants.UNIT_OF_MEASURE); + if (unit != null) unit = unit.trim(); + String actualComment = (unit != null && !unit.isEmpty()) ? "Unit: " + unit : null; + + IndicatorActivity ia = findExistingIndicatorActivity(activity, indicator, activityLocation); + if (ia != null) { + mergeIndicatorValuesIntoExisting(ia, session, config, + baseOrigVal, origBaseDate, baseRevVal, revBaseDate, + targetOrigVal, origTargetDate, targetRevVal, revTargetDate, + actualVal, actualDate, actualComment); + session.flush(); + return; + } + + ia = new IndicatorActivity(); + ia.setActivity(activity); + ia.setIndicator(indicator); + if (activityLocation != null) ia.setActivityLocation(activityLocation); Set values = new HashSet<>(); if (getKey(config, ImporterConstants.ORIGINAL_BASE_VALUE) != null || getKey(config, ImporterConstants.REVISED_BASE_VALUE) != null || existingBase != null) { @@ -1531,8 +1543,7 @@ public static void addIndicatorDataToActivity(Long activityId, Row row, Sheet sh actual.setValue(actualVal); actual.setValueDate(actualDate); actual.setIndicatorConnection(ia); - String unit = getCellValueByConfig(row, sheet, config, ImporterConstants.UNIT_OF_MEASURE); - if (unit != null && !unit.trim().isEmpty()) actual.setComment("Unit: " + unit.trim()); + if (actualComment != null) actual.setComment(actualComment); values.add(actual); ia.setValues(values); @@ -1542,6 +1553,135 @@ public static void addIndicatorDataToActivity(Long activityId, Row row, Sheet sh session.flush(); } + /** + * Finds an existing activity–indicator connection for the same activity, indicator, and location. + * Match is by indicator id and activity location (both null or same location). + */ + private static IndicatorActivity findExistingIndicatorActivity(AmpActivityVersion activity, AmpIndicator indicator, + AmpActivityLocation activityLocation) { + if (activity.getIndicators() == null) return null; + Long indicatorId = indicator != null ? indicator.getIndicatorId() : null; + Long locationId = activityLocation != null && activityLocation.getLocation() != null + ? activityLocation.getLocation().getAmpLocationId() : null; + for (IndicatorActivity ia : activity.getIndicators()) { + if (ia.getIndicator() == null) continue; + if (!Objects.equals(ia.getIndicator().getIndicatorId(), indicatorId)) continue; + Long existingLocId = ia.getActivityLocation() != null && ia.getActivityLocation().getLocation() != null + ? ia.getActivityLocation().getLocation().getAmpLocationId() : null; + if (Objects.equals(existingLocId, locationId)) return ia; + } + return null; + } + + /** + * Merges imported values into an existing indicator connection: updates existing values by type where present, + * adds new values only for types that are missing. + */ + private static void mergeIndicatorValuesIntoExisting(IndicatorActivity ia, Session session, Map config, + double baseOrigVal, Date origBaseDate, double baseRevVal, Date revBaseDate, + double targetOrigVal, Date origTargetDate, double targetRevVal, Date revTargetDate, + double actualVal, Date actualDate, String actualComment) { + Set existing = ia.getValues(); + if (existing == null) { + existing = new HashSet<>(); + ia.setValues(existing); + } + boolean hasBase = getKey(config, ImporterConstants.ORIGINAL_BASE_VALUE) != null || getKey(config, ImporterConstants.REVISED_BASE_VALUE) != null; + boolean hasTarget = getKey(config, ImporterConstants.ORIGINAL_TARGET_VALUE) != null || getKey(config, ImporterConstants.REVISED_TARGET_VALUE) != null; + + AmpIndicatorValue existingActual = findValueByType(existing, AmpIndicatorValue.ACTUAL); + if (existingActual != null) { + existingActual.setValue(actualVal); + existingActual.setValueDate(actualDate); + if (actualComment != null) existingActual.setComment(actualComment); + } else { + AmpIndicatorValue actual = new AmpIndicatorValue(AmpIndicatorValue.ACTUAL); + actual.setValue(actualVal); + actual.setValueDate(actualDate); + if (actualComment != null) actual.setComment(actualComment); + actual.setIndicatorConnection(ia); + existing.add(actual); + session.save(actual); + } + + if (hasBase) { + AmpIndicatorValue existingBase = findValueByType(existing, AmpIndicatorValue.BASE); + if (existingBase != null) { + existingBase.setValue(baseOrigVal); + existingBase.setValueDate(origBaseDate); + } else { + AmpIndicatorValue baseOrig = new AmpIndicatorValue(AmpIndicatorValue.BASE); + baseOrig.setValue(baseOrigVal); + baseOrig.setValueDate(origBaseDate); + baseOrig.setIndicatorConnection(ia); + existing.add(baseOrig); + session.save(baseOrig); + } + AmpIndicatorValue existingRev = findValueByType(existing, AmpIndicatorValue.REVISED); + if (existingRev != null) { + existingRev.setValue(baseRevVal); + existingRev.setValueDate(revBaseDate); + } else { + AmpIndicatorValue baseRev = new AmpIndicatorValue(AmpIndicatorValue.REVISED); + baseRev.setValue(baseRevVal); + baseRev.setValueDate(revBaseDate); + baseRev.setIndicatorConnection(ia); + existing.add(baseRev); + session.save(baseRev); + } + } + + if (hasTarget) { + List targets = getValuesByType(existing, AmpIndicatorValue.TARGET); + if (targets.size() >= 2) { + targets.get(0).setValue(targetOrigVal); + targets.get(0).setValueDate(origTargetDate); + targets.get(1).setValue(targetRevVal); + targets.get(1).setValueDate(revTargetDate); + } else if (targets.size() == 1) { + targets.get(0).setValue(targetOrigVal); + targets.get(0).setValueDate(origTargetDate); + AmpIndicatorValue tRev = new AmpIndicatorValue(AmpIndicatorValue.TARGET); + tRev.setValue(targetRevVal); + tRev.setValueDate(revTargetDate); + tRev.setIndicatorConnection(ia); + existing.add(tRev); + session.save(tRev); + } else { + AmpIndicatorValue tOrig = new AmpIndicatorValue(AmpIndicatorValue.TARGET); + tOrig.setValue(targetOrigVal); + tOrig.setValueDate(origTargetDate); + tOrig.setIndicatorConnection(ia); + existing.add(tOrig); + session.save(tOrig); + AmpIndicatorValue tRev = new AmpIndicatorValue(AmpIndicatorValue.TARGET); + tRev.setValue(targetRevVal); + tRev.setValueDate(revTargetDate); + tRev.setIndicatorConnection(ia); + existing.add(tRev); + session.save(tRev); + } + } + } + + private static AmpIndicatorValue findValueByType(Set values, int valueType) { + if (values == null) return null; + for (AmpIndicatorValue v : values) { + if (v.getValueType() == valueType) return v; + } + return null; + } + + private static List getValuesByType(Set values, int valueType) { + if (values == null) return Collections.emptyList(); + List list = new ArrayList<>(); + for (AmpIndicatorValue v : values) { + if (v.getValueType() == valueType) list.add(v); + } + list.sort(Comparator.comparing(AmpIndicatorValue::getValueDate, Comparator.nullsLast(Comparator.naturalOrder()))); + return list; + } + /** * Adds the program (theme) to the activity's programs if not already present. * When the Program Percentage field is enabled, percentages are recalculated and From bf722b0bde4694e21e064b0517defd8a4c64ed18 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Mon, 16 Feb 2026 15:34:06 +0300 Subject: [PATCH 09/59] Use indicator Location column Resolve missing created by --- .../dashboards/services/SectorSchemeDTO.java | 12 ++++++------ .../manager/IndicatorManagerService.java | 6 ++++++ .../action/dataimporter/util/ImporterUtil.java | 16 +++++++++++----- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/dashboards/services/SectorSchemeDTO.java b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/dashboards/services/SectorSchemeDTO.java index 0f062b3d42a..e143bf1e1a9 100644 --- a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/dashboards/services/SectorSchemeDTO.java +++ b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/dashboards/services/SectorSchemeDTO.java @@ -26,12 +26,12 @@ public class SectorSchemeDTO { private final SectorDTO[] children; public SectorSchemeDTO(AmpSectorScheme scheme, SectorDTO[] children) { - this.ampSecSchemeId = scheme.getAmpSecSchemeId(); - this.secSchemeCode = scheme.getSecSchemeCode(); - this.secSchemeName = scheme.getSecSchemeName(); - this.showInRMFilters = scheme.getShowInRMFilters(); - this.used = scheme.isUsed(); - this.children = children; + this.ampSecSchemeId = scheme != null ? scheme.getAmpSecSchemeId() : null; + this.secSchemeCode = scheme != null ? scheme.getSecSchemeCode() : null; + this.secSchemeName = scheme != null ? scheme.getSecSchemeName() : null; + this.showInRMFilters = Boolean.TRUE.equals(scheme != null ? scheme.getShowInRMFilters() : null); + this.used = scheme != null && scheme.isUsed(); + this.children = children != null ? children : new SectorDTO[0]; } public Long getAmpSecSchemeId() { diff --git a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/indicator/manager/IndicatorManagerService.java b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/indicator/manager/IndicatorManagerService.java index eefcfb1ee27..6e2339ae94b 100644 --- a/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/indicator/manager/IndicatorManagerService.java +++ b/amp/src/main/java/org/digijava/kernel/ampapi/endpoints/indicator/manager/IndicatorManagerService.java @@ -209,6 +209,12 @@ private void validateYear(MEIndicatorDTO value) { } private void validateYearRange(String startYear, String endYear, AmpIndicatorGlobalValue value, String error){ + if (value == null) { + return; + } + if (startYear == null || endYear == null) { + return; + } String startInString = "01/01/" + startYear; DateTime dateTime = DateTime.parse(startInString, formatter); diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java index ccf7936f9dd..a271a718199 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java @@ -689,7 +689,7 @@ public static AmpActivityVersion existingActivity(String projectTitle, String pr } // Prefer project code if provided if (projectCode != null && !projectCode.trim().isEmpty()) { - String hqlByCode = "SELECT a FROM " + AmpActivityVersion.class.getName() + " a WHERE a.projectCode = :projectCode"; + String hqlByCode = "SELECT a FROM " + AmpActivityVersion.class.getName() + " a LEFT JOIN FETCH a.activityCreator WHERE a.projectCode = :projectCode"; Query queryByCode = session.createQuery(hqlByCode); queryByCode.setCacheable(true); queryByCode.setParameter("projectCode", projectCode.trim(), StringType.INSTANCE); @@ -700,7 +700,7 @@ public static AmpActivityVersion existingActivity(String projectTitle, String pr } // Fall back to project title (name) if (projectTitle != null && !projectTitle.trim().isEmpty()) { - String hql = "SELECT a FROM " + AmpActivityVersion.class.getName() + " a WHERE a.name = :name"; + String hql = "SELECT a FROM " + AmpActivityVersion.class.getName() + " a LEFT JOIN FETCH a.activityCreator WHERE a.name = :name"; Query query = session.createQuery(hql); query.setCacheable(true); query.setParameter("name", projectTitle.trim(), StringType.INSTANCE); @@ -727,7 +727,8 @@ public static void setStatus(ImportDataModel importDataModel) { /** * Ensures created_by in the activity map is set to a valid team member id when null, * so the activity API validator does not reject with "(Invalid field value) created_by". - * For new activities uses current user; for updates uses existing activity's creator. + * For new activities uses current user; for updates uses existing activity's creator only + * when that creator is present (never overwrite with current user for existing activities). */ private static void ensureCreatedBySet(Map map, AmpActivityVersion existing) { Object createdBy = map.get(CREATED_BY_KEY); @@ -735,8 +736,13 @@ private static void ensureCreatedBySet(Map map, AmpActivityVersi return; } Long creatorId = null; - if (existing != null && existing.getActivityCreator() != null) { - creatorId = existing.getActivityCreator().getAmpTeamMemId(); + if (existing != null) { + AmpTeamMember creator = existing.getActivityCreator(); + if (creator != null) { + creatorId = creator.getAmpTeamMemId(); + } + // When existing activity has no creator (legacy data), leave created_by null; + // do not set to current user so we don't wrongly attribute creation. } else { AmpTeamMember currentMember = TeamUtil.getCurrentAmpTeamMember(); if (currentMember != null) { From b2acdf4112f76230033bbb5c337962911df15c07 Mon Sep 17 00:00:00 2001 From: brianbrix Date: Mon, 16 Feb 2026 15:47:10 +0300 Subject: [PATCH 10/59] Correct location ID --- .../module/aim/action/dataimporter/util/ImporterUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java index a271a718199..d7d3abaf5fe 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java @@ -1568,12 +1568,12 @@ private static IndicatorActivity findExistingIndicatorActivity(AmpActivityVersio if (activity.getIndicators() == null) return null; Long indicatorId = indicator != null ? indicator.getIndicatorId() : null; Long locationId = activityLocation != null && activityLocation.getLocation() != null - ? activityLocation.getLocation().getAmpLocationId() : null; + ? activityLocation.getLocation().getId() : null; for (IndicatorActivity ia : activity.getIndicators()) { if (ia.getIndicator() == null) continue; if (!Objects.equals(ia.getIndicator().getIndicatorId(), indicatorId)) continue; Long existingLocId = ia.getActivityLocation() != null && ia.getActivityLocation().getLocation() != null - ? ia.getActivityLocation().getLocation().getAmpLocationId() : null; + ? ia.getActivityLocation().getLocation().getId() : null; if (Objects.equals(existingLocId, locationId)) return ia; } return null; From f7eeee4b312c4e1d90714c7606e4cf651b6069cb Mon Sep 17 00:00:00 2001 From: brianbrix Date: Mon, 16 Feb 2026 16:30:38 +0300 Subject: [PATCH 11/59] Usse Project Location field instead of Location --- .../aim/action/dataimporter/DataImporter.java | 1 - .../action/dataimporter/ExcelImporter.java | 2 +- .../action/dataimporter/TxtDataImporter.java | 2 +- .../dataimporter/util/ImporterUtil.java | 88 +++++++++++++++---- .../jsp/aim/view/viewImportProgress.jsp | 8 +- 5 files changed, 78 insertions(+), 23 deletions(-) diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java index 4eff3282dec..9728d5b9dd2 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java @@ -501,7 +501,6 @@ private List getEntityFieldsInfo() { // Indicator columns for M&E import fieldsInfos.add(ImporterConstants.INDICATOR_NAME); fieldsInfos.add(ImporterConstants.PROGRAM_NAME); - fieldsInfos.add(ImporterConstants.LOCATION); fieldsInfos.add(ImporterConstants.INDICATOR_LOCATION); fieldsInfos.add(ImporterConstants.ORIGINAL_BASE_VALUE); fieldsInfos.add(ImporterConstants.ORIGINAL_BASE_VALUE_DATE); diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java index 7932735c045..ff0ac9637c9 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/ExcelImporter.java @@ -168,7 +168,7 @@ public static void processBatch(List batch,Sheet sheet, HttpServletRequest ImportDataModel importDataModel = new ImportDataModel(); importDataModel.setModified_by(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeamMemId()); - importDataModel.setCreated_by(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeamMemId()); + // created_by is set in ensureCreatedBySet when building the API map (correct for new vs existing) importDataModel.setTeam(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeam().getAmpTeamId()); importDataModel.setIs_draft(true); OffsetDateTime now = OffsetDateTime.now(ZoneOffset.UTC); diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java index 0453516a45c..12663cf8faa 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/TxtDataImporter.java @@ -93,7 +93,7 @@ private static void processBatch(List> batch, HttpServletRe ImportDataModel importDataModel = new ImportDataModel(); importDataModel.setModified_by(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeamMemId()); - importDataModel.setCreated_by(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeamMemId()); + // created_by is set in ensureCreatedBySet when building the API map (correct for new vs existing) importDataModel.setTeam(TeamMemberUtil.getCurrentAmpTeamMember(request).getAmpTeam().getAmpTeamId()); importDataModel.setIs_draft(true); OffsetDateTime now = OffsetDateTime.now(ZoneOffset.UTC); diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java index d7d3abaf5fe..b41a9fa4a60 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/util/ImporterUtil.java @@ -277,6 +277,10 @@ private static Session getSession() { private static String getFundingDate(String dateString) { + if (dateString != null && dateString.trim().matches("\\d{4}")) { + int year = Integer.parseInt(dateString.trim()); + return LocalDate.of(year, 1, 1).format(DateTimeFormatter.ofPattern("yyyy-MM-dd")); + } LocalDate date = LocalDate.now(); if (isCommonDateFormat(dateString)) { List formatters = Arrays.asList( @@ -731,26 +735,23 @@ public static void setStatus(ImportDataModel importDataModel) { * when that creator is present (never overwrite with current user for existing activities). */ private static void ensureCreatedBySet(Map map, AmpActivityVersion existing) { - Object createdBy = map.get(CREATED_BY_KEY); - if (createdBy != null) { - return; - } - Long creatorId = null; if (existing != null) { AmpTeamMember creator = existing.getActivityCreator(); - if (creator != null) { - creatorId = creator.getAmpTeamMemId(); - } - // When existing activity has no creator (legacy data), leave created_by null; - // do not set to current user so we don't wrongly attribute creation. - } else { - AmpTeamMember currentMember = TeamUtil.getCurrentAmpTeamMember(); - if (currentMember != null) { - creatorId = currentMember.getAmpTeamMemId(); + if (creator == null) { + // Existing activity has no creator (legacy); API expects null. + map.put(CREATED_BY_KEY, null); + return; } + map.put(CREATED_BY_KEY, creator.getAmpTeamMemId()); + return; } - if (creatorId != null) { - map.put(CREATED_BY_KEY, creatorId); + Object createdBy = map.get(CREATED_BY_KEY); + if (createdBy != null) { + return; + } + AmpTeamMember currentMember = TeamUtil.getCurrentAmpTeamMember(); + if (currentMember != null) { + map.put(CREATED_BY_KEY, currentMember.getAmpTeamMemId()); } } @@ -1200,6 +1201,58 @@ public static void updateLocations(ImportDataModel importDataModel, String locat } + /** + * Ensures the activity has an activity location for the given location name (for indicator location). + * If the location is not already on the activity, resolves it by name and adds it. + * @return the AmpActivityLocation for the name, or null if the location name cannot be resolved + */ + private static AmpActivityLocation getOrAddActivityLocationForName(AmpActivityVersion activity, String locationName, Session session) { + if (activity == null || locationName == null || locationName.trim().isEmpty()) return null; + locationName = locationName.trim(); + if (ConstantsMap.containsKey("location_" + locationName)) { + Long locationId = ConstantsMap.get("location_" + locationName); + AmpCategoryValueLocations loc = session.get(AmpCategoryValueLocations.class, locationId); + if (loc == null) return null; + AmpActivityLocation aal = new AmpActivityLocation(); + aal.setActivity(activity); + aal.setLocation(loc); + aal.setLocationPercentage(100f); + if (activity.getLocations() == null) activity.setLocations(new HashSet<>()); + activity.getLocations().add(aal); + session.save(aal); + return aal; + } + if (!session.isOpen()) { + session = PersistenceManager.getRequestDBSession(); + } + final Long[] foundId = new Long[1]; + session.doWork(connection -> { + String query = "SELECT acvl.id AS location_id FROM amp_category_value_location acvl WHERE LOWER(acvl.location_name) = LOWER(?)"; + try (PreparedStatement statement = connection.prepareStatement(query)) { + statement.setString(1, locationName); + try (ResultSet resultSet = statement.executeQuery()) { + if (resultSet.next()) { + foundId[0] = resultSet.getLong("location_id"); + ConstantsMap.put("location_" + locationName, foundId[0]); + } + } + } catch (SQLException e) { + logger.error("Error resolving location by name: " + locationName, e); + } + }); + if (foundId[0] == null) return null; + AmpCategoryValueLocations loc = session.get(AmpCategoryValueLocations.class, foundId[0]); + if (loc == null) return null; + AmpActivityLocation aal = new AmpActivityLocation(); + aal.setActivity(activity); + aal.setLocation(loc); + aal.setLocationPercentage(100f); + if (activity.getLocations() == null) activity.setLocations(new HashSet<>()); + activity.getLocations().add(aal); + session.save(aal); + return aal; + } + public static void updateImpLevels(ImportDataModel importDataModel, Session session) { if (ConstantsMap.containsKey("implementation_level_")) { @@ -1480,6 +1533,9 @@ public static void addIndicatorDataToActivity(Long activityId, Row row, Sheet sh } } } + if (activityLocation == null) { + activityLocation = getOrAddActivityLocationForName(activity, locationName, session); + } AmpIndicatorGlobalValue existingBase = indicator.getBaseValue(); double origBase = parseDoubleFromConfig(row, sheet, config, ImporterConstants.ORIGINAL_BASE_VALUE); diff --git a/amp/src/main/webapp/WEB-INF/jsp/aim/view/viewImportProgress.jsp b/amp/src/main/webapp/WEB-INF/jsp/aim/view/viewImportProgress.jsp index 3576600ac2b..0668bb28bb6 100644 --- a/amp/src/main/webapp/WEB-INF/jsp/aim/view/viewImportProgress.jsp +++ b/amp/src/main/webapp/WEB-INF/jsp/aim/view/viewImportProgress.jsp @@ -53,9 +53,9 @@ console.log("Response: " + JSON.stringify(response)); var data = JSON.parse(JSON.stringify(response)); $(".countRecords").html( - '

All Projects: ' +data.totalProjects+'

' + - '

Successful Projects: ' +data.successfulProjects+'

' + - '

Failed Projects: ' +data.failedProjects +'

' + '

All Records: ' +data.totalProjects+'

' + + '

Successful Records: ' +data.successfulProjects+'

' + + '

Failed Records: ' +data.failedProjects +'

' ); var importProjects = data.importedProjects; @@ -150,7 +150,7 @@
-

File Projects

+

File Records

From babda87c745b88140c5267c70104abc7ea012a0e Mon Sep 17 00:00:00 2001 From: brianbrix Date: Mon, 16 Feb 2026 16:38:42 +0300 Subject: [PATCH 12/59] Enable edit of column pair config --- .../aim/action/dataimporter/DataImporter.java | 75 ++++++++++++++++--- .../WEB-INF/jsp/aim/view/dataImporter.jsp | 47 +++++++----- 2 files changed, 95 insertions(+), 27 deletions(-) diff --git a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java index 9728d5b9dd2..a35bab8d5f0 100644 --- a/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java +++ b/amp/src/main/java/org/digijava/module/aim/action/dataimporter/DataImporter.java @@ -192,13 +192,20 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet String columnName = request.getParameter("columnName"); String selectedField = request.getParameter("selectedField"); - dataImporterForm.getColumnPairs().put(columnName, selectedField); - logger.info("Column Pairs:" + dataImporterForm.getColumnPairs()); + String configName = request.getParameter("configName"); + Map columnPairs = dataImporterForm.getColumnPairs(); + + if (configName != null && !configName.trim().isEmpty()) { + addColumnPairToConfig(configName.trim(), columnName, selectedField); + columnPairs = getConfigByName(configName.trim()); + } else { + columnPairs.put(columnName, selectedField); + } + logger.info("Column Pairs:" + columnPairs); ObjectMapper objectMapper = new ObjectMapper(); - String json = objectMapper.writeValueAsString(dataImporterForm.getColumnPairs()); + String json = objectMapper.writeValueAsString(columnPairs); - // Send response response.setContentType("application/json"); response.getWriter().write(json); response.setCharacterEncoding("UTF-8"); @@ -213,14 +220,20 @@ public ActionForward execute(ActionMapping mapping, ActionForm form, HttpServlet String columnName = request.getParameter("columnName"); String selectedField = request.getParameter("selectedField"); - dataImporterForm.getColumnPairs().put(columnName, selectedField); - removeMapItem(dataImporterForm.getColumnPairs(), columnName, selectedField); - logger.info("Column Pairs:" + dataImporterForm.getColumnPairs()); + String configName = request.getParameter("configName"); + Map columnPairs = dataImporterForm.getColumnPairs(); + + if (configName != null && !configName.trim().isEmpty()) { + removeColumnPairFromConfig(configName.trim(), columnName); + columnPairs = getConfigByName(configName.trim()); + } else { + removeMapItem(columnPairs, columnName, selectedField); + } + logger.info("Column Pairs:" + columnPairs); ObjectMapper objectMapper = new ObjectMapper(); - String json = objectMapper.writeValueAsString(dataImporterForm.getColumnPairs()); + String json = objectMapper.writeValueAsString(columnPairs); - // Send response response.setContentType("application/json"); response.setCharacterEncoding("UTF-8"); response.getWriter().write(json); @@ -400,6 +413,50 @@ private static Map getConfigByName(String configName) { return configValues; } + private static DataImporterConfig getConfigEntityByName(String configName) { + Session session = PersistenceManager.getRequestDBSession(); + String hql = "FROM DataImporterConfig WHERE configName = :configName"; + Query query = session.createQuery(hql); + query.setParameter("configName", configName, StringType.INSTANCE); + query.setMaxResults(1); + List list = query.list(); + return list.isEmpty() ? null : list.get(0); + } + + private static void addColumnPairToConfig(String configName, String columnName, String selectedField) { + DataImporterConfig config = getConfigEntityByName(configName); + if (config == null) { + logger.warn("Config not found for name: {}", configName); + return; + } + Session session = PersistenceManager.getRequestDBSession(); + DataImporterConfigValues cv = new DataImporterConfigValues(columnName, selectedField, config); + session.save(cv); + config.getConfigValues().add(cv); + session.flush(); + } + + private static void removeColumnPairFromConfig(String configName, String columnName) { + DataImporterConfig config = getConfigEntityByName(configName); + if (config == null) { + logger.warn("Config not found for name: {}", configName); + return; + } + DataImporterConfigValues toRemove = null; + for (DataImporterConfigValues v : config.getConfigValues()) { + if (columnName.equals(v.getConfigKey())) { + toRemove = v; + break; + } + } + if (toRemove != null) { + config.getConfigValues().remove(toRemove); + Session session = PersistenceManager.getRequestDBSession(); + session.delete(toRemove); + session.flush(); + } + } + public static void saveImportConfig(HttpServletRequest request, String fileName, Map config) { logger.info("Saving import config"); diff --git a/amp/src/main/webapp/WEB-INF/jsp/aim/view/dataImporter.jsp b/amp/src/main/webapp/WEB-INF/jsp/aim/view/dataImporter.jsp index ae5ab8a59d0..35c70be99ba 100644 --- a/amp/src/main/webapp/WEB-INF/jsp/aim/view/dataImporter.jsp +++ b/amp/src/main/webapp/WEB-INF/jsp/aim/view/dataImporter.jsp @@ -20,12 +20,18 @@ } } function addField() { - var columnName = document.getElementById("columnName").value; + var columnName = ($('#current-config-name').val()) + ? $('#column-name-edit').val() + : document.getElementById("columnName").value; var selectedField = document.getElementById("selected-field").value; - - sendValuesToBackend(columnName,selectedField,"addField") - - + if (!columnName || !selectedField) { + alert("Please enter column name and select a field."); + return; + } + sendValuesToBackend(columnName, selectedField, "addField"); + if ($('#current-config-name').val()) { + $('#column-name-edit').val(''); + } } $(document).ready(function() { $('#existing-config').val('0'); @@ -121,7 +127,7 @@ $('.existing-config').change(function() { var configName = $(this).val(); - if (configName!=='none'){ + if (configName!=='none' && configName!=='0'){ var formData = new FormData(); formData.append("configName", configName); @@ -138,33 +144,28 @@ console.log("Response: ",response); $("#templateUploadForm").hide(); $('#existing-config').val('0'); + $('#current-config-name').val(configName); return response.json(); }) .then(updatedMap => { console.log("Map :" ,updatedMap) - // Update UI or perform any additional actions if needed - console.log("Selected pairs updated successfully."); - console.log("Updated map received:", updatedMap); var tbody = document.getElementById("selected-pairs-table-body"); - - // Remove all rows from the table body tbody.innerHTML = ""; for (var key in updatedMap) { if (updatedMap.hasOwnProperty(key)) { - // Access each property using the key var value = updatedMap[key]; updateTable(key, value, tbody); console.log('Key:', key, 'Value:', value); } } document.getElementById("otherComponents").removeAttribute("hidden"); - $('#add-field').hide(); - $('.remove-row').hide(); - $('#selected-field').hide(); - $('#existing-config').val('1'); - + $('#add-field').show(); + $('.remove-row').show(); + $('#selected-field').show(); + $('#add-pair-edit-section').show(); + $('#column-name-edit').val(''); }) .catch(error => { @@ -172,6 +173,8 @@ }); }else { + $('#current-config-name').val(''); + $('#add-pair-edit-section').hide(); $("#templateUploadForm").show(); } }); @@ -179,11 +182,14 @@ function sendValuesToBackend(columnName, selectedField, action) { - // Create a FormData object to send data in the request body var formData = new FormData(); formData.append("columnName", columnName); formData.append("selectedField", selectedField); formData.append("action", action); + var configName = $('#current-config-name').val(); + if (configName) { + formData.append("configName", configName); + } fetch("${pageContext.request.contextPath}/aim/dataImporter.do", { method: "POST", @@ -474,11 +480,16 @@