excel-reader增加headerRowCount配置

entropy-cloud · Jan 9, 2025 · 8c8040c · 8c8040c
1 parent c9d63d2
commit 8c8040c
Show file tree

Hide file tree

Showing 15 changed files with 235 additions and 23 deletions.
diff --git a/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/manager/FileBatchSupport.java b/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/manager/FileBatchSupport.java
@@ -2,11 +2,13 @@
 
 import io.nop.api.core.ioc.IBeanProvider;
 import io.nop.batch.core.IBatchAggregator;
+import io.nop.batch.core.IBatchChunkContext;
 import io.nop.batch.core.IBatchLoaderProvider;
 import io.nop.batch.core.IBatchRecordFilter;
 import io.nop.batch.core.IBatchTaskContext;
 import io.nop.batch.core.consumer.ResourceRecordConsumerProvider;
 import io.nop.batch.core.filter.EvalBatchRecordFilter;
+import io.nop.batch.core.loader.ListBatchLoader;
 import io.nop.batch.core.loader.ResourceRecordLoaderProvider;
 import io.nop.batch.dsl.BatchDslConstants;
 import io.nop.batch.dsl.model.BatchExcelReaderModel;
@@ -25,8 +27,10 @@
 import io.nop.report.core.record.ExcelResourceIO;
 import io.nop.xlang.api.XLang;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 
 public class FileBatchSupport {
 
@@ -77,7 +81,7 @@ private static IResourceRecordInputProvider<Object> newRecordInputProvider(Batch
         io.setHeaders(readerModel.getHeaders());
         io.setHeaderLabels(readerModel.getHeaderLabels());
         if (readerModel.getCsvFormat() != null) {
-           io.setFormat(readerModel.getCsvFormat());
+            io.setFormat(readerModel.getCsvFormat());
         }
         if (readerModel.getHeadersNormalizer() != null) {
             io.setHeadersNormalizer(headers -> (List<String>) readerModel.getHeadersNormalizer().call1(null, headers, XLang.newEvalScope()));
@@ -126,7 +130,7 @@ private static IResourceRecordOutputProvider<Object> newRecordOutputProvider(Bat
         io.setHeaders(writerModel.getHeaders());
         io.setHeaderLabels(writerModel.getHeaderLabels());
         if (writerModel.getCsvFormat() != null) {
-           io.setFormat(writerModel.getCsvFormat());
+            io.setFormat(writerModel.getCsvFormat());
         }
         return io;
     }
@@ -172,9 +176,37 @@ private static ExcelResourceIO<Object> newExcelIO(IBatchExcelIOModel ioModel) {
         config.setHeaderSheetName(ioModel.getHeaderSheetName());
         config.setTrailerSheetName(ioModel.getTrailerSheetName());
         config.setTemplatePath(ioModel.getTemplatePath());
+        if (ioModel.getHeaderRowCount() != null) {
+            config.setHeaderRowCount(ioModel.getHeaderRowCount());
+        }
+
         io.setIOConfig(config);
         io.setHeaders(ioModel.getHeaders());
         io.setHeaderLabels(ioModel.getHeaderLabels());
         return io;
     }
-}
+
+    static final String KEY_LOADER = "batchLoader";
+
+    public static <T> List<T> batchLoad(int batchSize, IBatchChunkContext chunkCtx, IBatchLoaderProvider<T> provider) {
+        IBatchTaskContext taskCtx = chunkCtx.getTaskContext();
+        synchronized (chunkCtx) {
+            IBatchLoaderProvider.IBatchLoader loader = (IBatchLoaderProvider.IBatchLoader) taskCtx.getAttribute(KEY_LOADER);
+            if (loader == null) {
+                loader = provider.setup(taskCtx);
+                taskCtx.setAttribute(KEY_LOADER, loader);
+            }
+            return loader.load(batchSize, chunkCtx);
+        }
+    }
+
+    public static <T> List<T> batchLoadList(int batchSize, IBatchChunkContext chunkCtx,
+                                            Function<IBatchTaskContext, List<T>> listProvider) {
+        return batchLoad(batchSize, chunkCtx, taskCtx -> {
+            List<T> list = listProvider.apply(taskCtx);
+            if (list == null)
+                list = Collections.emptyList();
+            return new ListBatchLoader<>(list);
+        });
+    }
+}
diff --git a/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/BatchExcelWriterModel.java b/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/BatchExcelWriterModel.java
@@ -1,9 +1,15 @@
 package io.nop.batch.dsl.model;
 
 import io.nop.batch.dsl.model._gen._BatchExcelWriterModel;
+import io.nop.core.lang.eval.IEvalFunction;
 
 public class BatchExcelWriterModel extends _BatchExcelWriterModel implements IBatchExcelIOModel {
     public BatchExcelWriterModel() {
 
     }
+
+    @Override
+    public IEvalFunction getHeadersNormalizer() {
+        return null;
+    }
 }
diff --git a/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/IBatchExcelIOModel.java b/nop-batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/IBatchExcelIOModel.java
@@ -1,6 +1,7 @@
 package io.nop.batch.dsl.model;
 
 import io.nop.core.lang.eval.IEvalAction;
+import io.nop.core.lang.eval.IEvalFunction;
 
 import java.util.List;
 
@@ -18,4 +19,8 @@ public interface IBatchExcelIOModel {
     List<String> getHeaders();
 
     List<String> getHeaderLabels();
+
+    Integer getHeaderRowCount();
+
+    IEvalFunction getHeadersNormalizer();
 }
diff --git a/...batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/_gen/_BatchExcelReaderModel.java b/...batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/_gen/_BatchExcelReaderModel.java
@@ -44,6 +44,13 @@ public abstract class _BatchExcelReaderModel extends io.nop.core.resource.compon
      */
     private java.util.List<java.lang.String> _headerLabels ;
 
+    /**
+     *  
+     * xml name: headerRowCount
+     * 
+     */
+    private java.lang.Integer _headerRowCount ;
+
     /**
      *  
      * xml name: headerSheetName
@@ -155,6 +162,25 @@ public void setHeaderLabels(java.util.List<java.lang.String> value){
     }
 
 
+    /**
+     * 
+     * xml name: headerRowCount
+     *  
+     */
+
+    public java.lang.Integer getHeaderRowCount(){
+      return _headerRowCount;
+    }
+
+
+    public void setHeaderRowCount(java.lang.Integer value){
+        checkAllowChange();
+
+        this._headerRowCount = value;
+
+    }
+
+
     /**
      * 
      * xml name: headerSheetName
@@ -269,6 +295,7 @@ protected void outputJson(IJsonHandler out){
         out.putNotNull("filePath",this.getFilePath());
         out.putNotNull("filter",this.getFilter());
         out.putNotNull("headerLabels",this.getHeaderLabels());
+        out.putNotNull("headerRowCount",this.getHeaderRowCount());
         out.putNotNull("headerSheetName",this.getHeaderSheetName());
         out.putNotNull("headers",this.getHeaders());
         out.putNotNull("headersNormalizer",this.getHeadersNormalizer());
@@ -289,6 +316,7 @@ protected void copyTo(BatchExcelReaderModel instance){
         instance.setFilePath(this.getFilePath());
         instance.setFilter(this.getFilter());
         instance.setHeaderLabels(this.getHeaderLabels());
+        instance.setHeaderRowCount(this.getHeaderRowCount());
         instance.setHeaderSheetName(this.getHeaderSheetName());
         instance.setHeaders(this.getHeaders());
         instance.setHeadersNormalizer(this.getHeadersNormalizer());

diff --git a/...batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/_gen/_BatchExcelWriterModel.java b/...batch/nop-batch-dsl/src/main/java/io/nop/batch/dsl/model/_gen/_BatchExcelWriterModel.java
@@ -37,6 +37,13 @@ public abstract class _BatchExcelWriterModel extends io.nop.core.resource.compon
      */
     private java.util.List<java.lang.String> _headerLabels ;
 
+    /**
+     *  
+     * xml name: headerRowCount
+     * 
+     */
+    private java.lang.Integer _headerRowCount ;
+
     /**
      *  
      * xml name: headerSheetName
@@ -122,6 +129,25 @@ public void setHeaderLabels(java.util.List<java.lang.String> value){
     }
 
 
+    /**
+     * 
+     * xml name: headerRowCount
+     *  
+     */
+
+    public java.lang.Integer getHeaderRowCount(){
+      return _headerRowCount;
+    }
+
+
+    public void setHeaderRowCount(java.lang.Integer value){
+        checkAllowChange();
+
+        this._headerRowCount = value;
+
+    }
+
+
     /**
      * 
      * xml name: headerSheetName
@@ -216,6 +242,7 @@ protected void outputJson(IJsonHandler out){
         out.putNotNull("dataSheetName",this.getDataSheetName());
         out.putNotNull("filePath",this.getFilePath());
         out.putNotNull("headerLabels",this.getHeaderLabels());
+        out.putNotNull("headerRowCount",this.getHeaderRowCount());
         out.putNotNull("headerSheetName",this.getHeaderSheetName());
         out.putNotNull("headers",this.getHeaders());
         out.putNotNull("templatePath",this.getTemplatePath());
@@ -234,6 +261,7 @@ protected void copyTo(BatchExcelWriterModel instance){
         instance.setDataSheetName(this.getDataSheetName());
         instance.setFilePath(this.getFilePath());
         instance.setHeaderLabels(this.getHeaderLabels());
+        instance.setHeaderRowCount(this.getHeaderRowCount());
         instance.setHeaderSheetName(this.getHeaderSheetName());
         instance.setHeaders(this.getHeaders());
         instance.setTemplatePath(this.getTemplatePath());

diff --git a/nop-commons/src/main/java/io/nop/commons/util/FileHelper.java b/nop-commons/src/main/java/io/nop/commons/util/FileHelper.java
@@ -36,6 +36,7 @@
 import java.util.Objects;
 import java.util.Properties;
 import java.util.Set;
+import java.util.function.BiFunction;
 import java.util.function.Function;
 
 import static io.nop.commons.CommonErrors.ARG_DEST;
@@ -549,4 +550,36 @@ public static FileVisitResult walk(File file, Function<File, FileVisitResult> fn
         }
         return FileVisitResult.SKIP_SUBTREE;
     }
+
+    public static FileVisitResult walk2(File dir1, File dir2, BiFunction<File, File, FileVisitResult> fn) {
+        FileVisitResult result = fn.apply(dir1, dir2);
+        if (result != null && result != FileVisitResult.CONTINUE)
+            return result;
+
+        File[] subFiles = dir1.listFiles();
+        if (subFiles != null) {
+            for (File subFile : subFiles) {
+                result = walk2(subFile, new File(dir2, subFile.getName()), fn);
+
+                if (result == FileVisitResult.SKIP_SIBLINGS)
+                    break;
+
+                if (result == FileVisitResult.TERMINATE)
+                    return result;
+            }
+        }
+        return FileVisitResult.SKIP_SUBTREE;
+    }
+
+    public static void copyFileWithFilter(File dir1, File dir2, BiFunction<File, File, Boolean> filter) {
+        walk2(dir1, dir2, (file1, file2) -> {
+            if (file1.isFile()) {
+                if (filter.apply(file1, file2)) {
+                    copyFile(file1, file2);
+                    return FileVisitResult.SKIP_SUBTREE;
+                }
+            }
+            return FileVisitResult.CONTINUE;
+        });
+    }
 }
diff --git a/nop-core/src/main/java/io/nop/core/CoreErrors.java b/nop-core/src/main/java/io/nop/core/CoreErrors.java
@@ -140,6 +140,8 @@ public interface CoreErrors {
     String ARG_VALUE = "value";
     String ARG_PATTERN = "pattern";
 
+    String ARG_TPL_PATH = "tplPath";
+
     String ARG_LOC_A = "locA";
     String ARG_LOC_B = "locB";
     String ARG_VALUE_A = "valueA";
@@ -529,6 +531,9 @@ public interface CoreErrors {
     ErrorCode ERR_RESOURCE_STORE_NOT_SUPPORT_TENANT_DELTA = define("nop.err.core.resource.store-not-support-tenant-delta",
             "文件存储没有开启租户Delta支持");
 
+    ErrorCode ERR_RESOURCE_TPL_NOT_EXISTS = define("nop.err.core.resource.tpl-not-exists",
+            "模板文件不存在:{tpl}", ARG_TPL_PATH);
+
     ErrorCode ERR_XML_NOT_NODE_VALUE = define("nop.err.core.xml.not-node-value",
             "值不是XNode类型:{value}", ARG_VALUE);
     ErrorCode ERR_XML_EXCEED_MAX_NESTED_LEVEL = define("nop.err.core.xml.exceed-max-nested-level", "XML嵌套层次超过限制");

diff --git a/nop-core/src/main/java/io/nop/core/resource/record/list/HeaderListRecordOutput.java b/nop-core/src/main/java/io/nop/core/resource/record/list/HeaderListRecordOutput.java
@@ -9,6 +9,8 @@
 
 import io.nop.commons.util.CollectionHelper;
 import io.nop.commons.util.StringHelper;
+import io.nop.core.lang.eval.EvalExprProvider;
+import io.nop.core.lang.eval.IEvalFunction;
 import io.nop.dataset.record.IRecordOutput;
 
 import java.util.ArrayList;
@@ -21,13 +23,25 @@ public class HeaderListRecordOutput<R> implements IRecordOutput<List<Object>> {
     private final BiFunction<List<String>, List<Object>, R> rowBuilder;
     private boolean headersWritten;
     private long writeCount;
-    private int skipCount;
+    private int headerRowCount;
+    private List<String> headerLabels;
+    private IEvalFunction headersNormalizer;
 
-    public HeaderListRecordOutput(int skipCount, BiFunction<List<String>, List<Object>, R> rowBuilder) {
-        this.skipCount = skipCount;
+    private List<List<String>> headerRows = new ArrayList<>();
+
+    public HeaderListRecordOutput(int headerRowCount, BiFunction<List<String>, List<Object>, R> rowBuilder) {
+        this.headerRowCount = headerRowCount;
         this.rowBuilder = rowBuilder;
     }
 
+    public void setHeaderLabels(List<String> headerLabels) {
+        this.headerLabels = headerLabels;
+    }
+
+    public void setHeadersNormalizer(IEvalFunction headersNormalizer) {
+        this.headersNormalizer = headersNormalizer;
+    }
+
     public List<R> getResult() {
         return records;
     }
@@ -38,21 +52,33 @@ public long getWriteCount() {
 
     @Override
     public void write(List<Object> record) {
-        if (skipCount > writeCount) {
+        if (headerRowCount > writeCount) {
+            List<String> row = CollectionHelper.toStringList(record);
+            headerRows.add(row);
             writeCount++;
             return;
         }
 
         if (!headersWritten) {
-            this.headers = CollectionHelper.toStringList(record);
+            normalizeHeaders();
             headersWritten = true;
-            return;
         }
 
         doWriteRecord(record);
         writeCount++;
     }
 
+    private void normalizeHeaders() {
+        if (this.headers == null || this.headers.isEmpty()) {
+            if (!headerRows.isEmpty())
+                this.headers = headerRows.get(headerRows.size() - 1);
+
+            if (headersNormalizer != null) {
+                this.headers = (List<String>) this.headersNormalizer.call2(null, headers, headerRows, EvalExprProvider.newEvalScope());
+            }
+        }
+    }
+
     private String toString(Object value) {
         return StringHelper.toString(value, "");
     }