diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/creditcardvalidator.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/creditcardvalidator.adoc index dbe19b40b6f..5be35e9dc71 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/creditcardvalidator.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/creditcardvalidator.adoc @@ -30,6 +30,8 @@ The Credit Card Validator transform will help you check the following: == Options +=== Fields tab + * transform name: the transform name, unique in a pipeline * Credit card field: the name of the input field that will contain the credit card number during execution * Get only digits? @@ -37,4 +39,34 @@ The Credit Card Validator transform will help you check the following: * Output Fields ** Result fieldname: the name of the (Boolean) output field indicating the validity of the number ** Credit card type field: the name of the output field that will hold the credit card type (vendor) -** Not valid message: the name of the output field that will hold the error message. +** Not valid message: the name of the output field that will hold the error message. When the BIN database recognizes a card type but the number fails validation, the message is `Not a valid number for {card type}.` + +=== BIN Files tab + +The BIN Files tab enables an optional lookup against an external Bank Identification Number (BIN) CSV database. When disabled, the built-in card type detection is used. + +* Use BIN database: enable to validate the card against an external CSV BIN database +* BIN database file: the CSV file containing the BIN database (browsable) +* Delimiter: the delimiter character used in the CSV file (default `,`) +* Enclosure: the enclosure (quote) character used in the CSV file (default `"`) +* Encoding: the character encoding of the CSV file (default UTF-8) +* Header row present: when enabled, the first row of the CSV is treated as a header. When disabled, columns are mapped by fixed position (`BIN, card type, issuer, country`). +* BIN CSV Column: the CSV header column that contains the BIN values. When empty, the transform tries to auto-detect a `bin` column. +* Get columns: reads the CSV header and populates the BIN CSV Column dropdown and the extra output fields. +* Extra output fields: a table mapping CSV columns to output fields. +** Field name: the name of the output field +** CSV column: the CSV column to map from +** Type: the Hop data type of the output field +** Format: the conversion mask +** Length: the field length +** Precision: the field precision +** Currency, Decimal, Group: the numeric/currency symbols +** Trim: the trim type + +The card number length is validated against a built-in card scheme table (Visa, Mastercard, American Express, Discover, JCB, Diners Club, UnionPay, Maestro, Verve), independent of the CSV file. + +NOTE: Only the BIN column is required in the CSV. The card type, issuer and country columns are optional and are detected by common column names such as `cardtype`, `issuer`, `country` when not explicitly mapped. + +NOTE: BIN values are limited to 8 digits. Rows whose BIN is empty, longer than 8 digits, or non-numeric are skipped, and the number of skipped rows is logged when the transform initializes. + +NOTE: When you uncheck *Header row present* while header-based column mappings (the *BIN CSV Column* or any extra output *CSV column*) are still configured, you are prompted to confirm. Confirming clears those mappings and switches to fixed-position mapping; cancelling keeps the header mode and its mappings. diff --git a/plugins/transforms/creditcardvalidator/pom.xml b/plugins/transforms/creditcardvalidator/pom.xml index 201ae65ad06..3c0bb7397a9 100644 --- a/plugins/transforms/creditcardvalidator/pom.xml +++ b/plugins/transforms/creditcardvalidator/pom.xml @@ -29,4 +29,47 @@ jar Hop Plugins Transforms Credit Card Validator + + 5.12.0 + + + + + + org.apache.hop + hop-libs + ${project.version} + pom + import + + + + + + + com.opencsv + opencsv + ${opencsv.version} + + + commons-beanutils + * + + + org.apache.commons + commons-collections4 + + + org.apache.commons + commons-lang3 + + + + + it.unimi.dsi + fastutil + 8.5.15 + + + diff --git a/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/BinDatabase.java b/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/BinDatabase.java new file mode 100644 index 00000000000..4ce700074e7 --- /dev/null +++ b/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/BinDatabase.java @@ -0,0 +1,344 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.creditcardvalidator; + +import com.opencsv.CSVParser; +import com.opencsv.CSVParserBuilder; +import com.opencsv.CSVReader; +import com.opencsv.CSVReaderBuilder; +import com.opencsv.exceptions.CsvValidationException; +import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.vfs.HopVfs; +import org.apache.hop.i18n.BaseMessages; + +/** In-memory lookup table for Bank Identification Numbers (BIN) loaded from an external CSV. */ +public class BinDatabase { + + private static final Class PKG = CreditCardValidatorMeta.class; + + private static final int MIN_BIN_LENGTH = 1; + private static final int MAX_BIN_LENGTH = 8; + private static final int INITIAL_CAPACITY = 10_000; + private static final Map EMPTY_VALUES = Collections.emptyMap(); + + private final Long2ObjectOpenHashMap records; + private final Map dedup = new HashMap<>(); + + /** Bitmask of the BIN lengths present in the dataset (bit {@code 1 << length}). */ + private int binLengths; + + /** Number of rows skipped because the BIN was empty, longer than 8 digits, or non-numeric. */ + private int skippedRows; + + public BinDatabase() { + this.records = new Long2ObjectOpenHashMap<>(INITIAL_CAPACITY); + } + + /** Load a user-supplied CSV file with an explicit column mapping and read configuration. */ + public void load( + IVariables variables, + String vfsFilename, + String binCsvColumn, + List outputFields, + String delimiter, + String enclosure, + String encoding, + boolean headerPresent) + throws HopException { + Charset charset; + try { + charset = Charset.forName(encoding == null ? "UTF-8" : encoding); + } catch (Exception e) { + charset = StandardCharsets.UTF_8; + } + try (InputStream in = HopVfs.getInputStream(vfsFilename, variables)) { + List resolvedFields = resolveOutputFields(outputFields, variables); + load(in, binCsvColumn, resolvedFields, delimiter, enclosure, charset, headerPresent); + } catch (IOException e) { + throw new HopException( + BaseMessages.getString(PKG, "CreditCardValidator.Error.BinDatabaseRead"), e); + } + } + + private static List resolveOutputFields( + List outputFields, IVariables variables) { + List resolved = new java.util.ArrayList<>(outputFields.size()); + for (BinOutputField outputField : outputFields) { + BinOutputField copy = outputField.clone(); + copy.setName(variables.resolve(outputField.getName())); + resolved.add(copy); + } + return resolved; + } + + private void load( + InputStream in, + String binCsvColumn, + List outputFields, + String delimiter, + String enclosure, + Charset charset, + boolean headerPresent) + throws HopException { + try (CSVReader reader = + new CSVReaderBuilder(new InputStreamReader(in, charset)) + .withCSVParser(parser(delimiter, enclosure)) + .build()) { + int binIdx; + int cardTypeIdx; + Map extraIdx = new HashMap<>(); + + if (headerPresent) { + String[] header = reader.readNext(); + if (header == null) { + throw new HopException( + BaseMessages.getString(PKG, "CreditCardValidator.Error.BinDatabaseEmpty")); + } + binIdx = resolveColumn(header, binCsvColumn, 0, "bin"); + if (binIdx < 0) { + throw new HopException( + BaseMessages.getString( + PKG, "CreditCardValidator.Error.BinColumnNotFound", binCsvColumn)); + } + cardTypeIdx = + resolveColumn( + header, null, "cardtype", "card_type", "card type", "brand", "scheme", "type"); + for (BinOutputField outputField : outputFields) { + if (Utils.isEmpty(outputField.getColumn())) { + continue; + } + for (int i = 0; i < header.length; i++) { + if (outputField.getColumn().equals(header[i])) { + extraIdx.put(outputField.getName(), i); + break; + } + } + } + } else { + binIdx = 0; + cardTypeIdx = 1; + for (int k = 0; k < outputFields.size(); k++) { + BinOutputField outputField = outputFields.get(k); + if (!Utils.isEmpty(outputField.getName())) { + extraIdx.put(outputField.getName(), 4 + k); + } + } + } + + String[] fields; + while ((fields = reader.readNext()) != null) { + String rawBin = field(fields, binIdx).trim(); + if (rawBin.isEmpty() || rawBin.length() > MAX_BIN_LENGTH) { + skippedRows++; + continue; + } + int bin = parseDigits(rawBin); + if (bin < 0) { + skippedRows++; + continue; + } + int length = rawBin.length(); + String cardType = dedup(field(fields, cardTypeIdx)); + Map extraValues = EMPTY_VALUES; + if (!extraIdx.isEmpty()) { + extraValues = new HashMap<>(); + for (Map.Entry entry : extraIdx.entrySet()) { + extraValues.put(entry.getKey(), field(fields, entry.getValue()).trim()); + } + } + + records.put(composeKey(length, bin), new BinRecord(cardType, extraValues)); + binLengths |= 1 << length; + } + } catch (IOException e) { + throw new HopException( + BaseMessages.getString(PKG, "CreditCardValidator.Error.BinDatabaseRead"), e); + } catch (CsvValidationException e) { + throw new HopException( + BaseMessages.getString(PKG, "CreditCardValidator.Error.BinDatabaseRead"), e); + } + } + + private static CSVParser parser(String delimiter, String enclosure) { + CSVParserBuilder builder = new CSVParserBuilder(); + if (!Utils.isEmpty(delimiter)) { + builder.withSeparator(delimiter.charAt(0)); + } + if (!Utils.isEmpty(enclosure)) { + builder.withQuoteChar(enclosure.charAt(0)); + } + return builder.build(); + } + + private static String field(String[] fields, int index) { + if (index < 0 || index >= fields.length || fields[index] == null) { + return ""; + } + return fields[index]; + } + + private String dedup(String value) { + String trimmed = value.trim(); + String existing = dedup.get(trimmed); + if (existing != null) { + return existing; + } + dedup.put(trimmed, trimmed); + return trimmed; + } + + /** Compose a unique long key from a BIN length and its numeric value. */ + private static long composeKey(int length, int value) { + return ((long) length << 32) | (value & 0xFFFFFFFFL); + } + + /** Parse a pure digit string into an int; returns -1 if any non-digit is found. */ + private static int parseDigits(String s) { + int value = 0; + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c < '0' || c > '9') { + return -1; + } + value = value * 10 + (c - '0'); + } + return value; + } + + /** Resolve the record matching the BIN using longest-prefix matching. */ + public BinRecord lookup(String bin) { + if (Utils.isEmpty(bin)) { + return null; + } + int len = bin.length(); + if (len > MAX_BIN_LENGTH) { + len = MAX_BIN_LENGTH; + } + for (int l = len; l >= MIN_BIN_LENGTH; l--) { + if ((binLengths & (1 << l)) == 0) { + continue; + } + BinRecord record = records.get(composeKey(l, prefixValue(bin, l))); + if (record != null) { + return record; + } + } + return null; + } + + private static int prefixValue(String bin, int length) { + int value = 0; + for (int i = 0; i < length; i++) { + char c = bin.charAt(i); + if (c < '0' || c > '9') { + return -1; + } + value = value * 10 + (c - '0'); + } + return value; + } + + public int size() { + return records.size(); + } + + public int getSkippedRows() { + return skippedRows; + } + + /** + * Resolve a column to its index. An explicit column name wins; otherwise the header is scanned + * for known names. When nothing matches, the default index is returned. + */ + private static int resolveColumn( + String[] header, String columnName, int defaultIndex, String... knownNames) { + int index = resolveColumn(header, columnName, knownNames); + return index < 0 ? defaultIndex : index; + } + + /** Resolve a column to its index; returns -1 when nothing matches. */ + private static int resolveColumn(String[] header, String columnName, String... knownNames) { + if (!Utils.isEmpty(columnName)) { + for (int i = 0; i < header.length; i++) { + if (columnName.equals(header[i])) { + return i; + } + } + return -1; + } + for (String knownName : knownNames) { + String normalized = normalize(knownName); + for (int i = 0; i < header.length; i++) { + if (normalized.equals(normalize(header[i]))) { + return i; + } + } + } + return -1; + } + + private static String normalize(String value) { + if (value == null) { + return ""; + } + StringBuilder sb = new StringBuilder(value.length()); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + if (c == ' ' || c == '_' || c == '-') { + continue; + } + sb.append(Character.toLowerCase(c)); + } + return sb.toString(); + } + + /** A single BIN entry. */ + public static class BinRecord { + private final String cardType; + private final Map extraValues; + + public BinRecord(String cardType, Map extraValues) { + this.cardType = cardType; + this.extraValues = extraValues; + } + + public String getCardType() { + return cardType; + } + + public Map getExtraValues() { + return extraValues; + } + + public String getValue(String name) { + return extraValues == null ? null : extraValues.get(name); + } + } +} diff --git a/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/BinOutputField.java b/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/BinOutputField.java new file mode 100644 index 00000000000..458cf4234c7 --- /dev/null +++ b/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/BinOutputField.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.creditcardvalidator; + +import java.util.Objects; +import lombok.Getter; +import lombok.Setter; +import org.apache.hop.core.exception.HopPluginException; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.value.ValueMetaFactory; +import org.apache.hop.metadata.api.HopMetadataProperty; + +/** Describes an output field mapped from a CSV column of the BIN file. */ +@Getter +@Setter +public class BinOutputField { + + @HopMetadataProperty(key = "name") + private String name; + + @HopMetadataProperty(key = "column") + private String column; + + @HopMetadataProperty(key = "type") + private int type = IValueMeta.TYPE_STRING; + + @HopMetadataProperty(key = "format") + private String format; + + @HopMetadataProperty(key = "length") + private int length = -1; + + @HopMetadataProperty(key = "precision") + private int precision = -1; + + @HopMetadataProperty(key = "currency") + private String currency; + + @HopMetadataProperty(key = "decimal") + private String decimal; + + @HopMetadataProperty(key = "group") + private String group; + + @HopMetadataProperty(key = "trim_type") + private int trimType = IValueMeta.TRIM_TYPE_NONE; + + public BinOutputField() {} + + public BinOutputField(String name, String column) { + this.name = name; + this.column = column; + } + + public BinOutputField(BinOutputField other) { + this.name = other.name; + this.column = other.column; + this.type = other.type; + this.format = other.format; + this.length = other.length; + this.precision = other.precision; + this.currency = other.currency; + this.decimal = other.decimal; + this.group = other.group; + this.trimType = other.trimType; + } + + @Override + public BinOutputField clone() { + return new BinOutputField(this); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof BinOutputField that)) { + return false; + } + return Objects.equals(name, that.name) && Objects.equals(column, that.column); + } + + @Override + public int hashCode() { + return Objects.hash(name, column); + } + + /** + * Build the value metadata for this output field, copying type, length, precision and symbols. + */ + public IValueMeta createValueMeta(String fieldName) throws HopPluginException { + IValueMeta valueMeta = ValueMetaFactory.createValueMeta(fieldName, type, length, precision); + valueMeta.setConversionMask(format); + valueMeta.setDecimalSymbol(decimal); + valueMeta.setGroupingSymbol(group); + valueMeta.setCurrencySymbol(currency); + valueMeta.setTrimType(trimType); + return valueMeta; + } +} diff --git a/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CardScheme.java b/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CardScheme.java new file mode 100644 index 00000000000..420caaf9b2c --- /dev/null +++ b/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CardScheme.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.creditcardvalidator; + +/** Built-in card scheme rules used to validate card length independently of the BIN file. */ +public final class CardScheme { + + private CardScheme() {} + + /** Checks whether the given card-number length is valid for the given BIN prefix. */ + public static boolean isValidLength(String bin, int cardLength) { + String b = bin == null ? "" : bin; + if (b.startsWith("4")) { + // Visa: 13, 16 or 19 + return cardLength == 13 || cardLength == 16 || cardLength == 19; + } + if (b.startsWith("34") || b.startsWith("37")) { + // American Express: 15 + return cardLength == 15; + } + if (b.startsWith("6011") || b.startsWith("65")) { + // Discover (6011, 65): 16 to 19 + return cardLength >= 16 && cardLength <= 19; + } + if (b.startsWith("644") + || b.startsWith("645") + || b.startsWith("646") + || b.startsWith("647") + || b.startsWith("648") + || b.startsWith("649")) { + // Discover (644-649): 16 to 19 + return cardLength >= 16 && cardLength <= 19; + } + if (b.length() >= 3 && b.charAt(0) == '6' && b.charAt(1) == '2') { + // Discover (622126-622925): 16 to 19 + return cardLength >= 16 && cardLength <= 19; + } + if (inRange(b, "2221", "2720")) { + // Mastercard (2221-2720): 16 + return cardLength == 16; + } + if (inRange(b, "51", "55")) { + // Mastercard (51-55): 16 + return cardLength == 16; + } + if (inRange(b, "3528", "3589")) { + // JCB (3528-3589): 16 to 19 + return cardLength >= 16 && cardLength <= 19; + } + if (b.startsWith("300") + || b.startsWith("301") + || b.startsWith("302") + || b.startsWith("303") + || b.startsWith("304") + || b.startsWith("305") + || b.startsWith("3095") + || b.startsWith("36") + || b.startsWith("38") + || b.startsWith("39")) { + // Diners Club: 14 to 19 + return cardLength >= 14 && cardLength <= 19; + } + if (b.startsWith("62")) { + // UnionPay: 16 to 19 + return cardLength >= 16 && cardLength <= 19; + } + if (b.startsWith("5018") + || b.startsWith("5020") + || b.startsWith("5038") + || b.startsWith("5893") + || b.startsWith("6304") + || b.startsWith("6759") + || b.startsWith("6761") + || b.startsWith("6762") + || b.startsWith("6763")) { + // Maestro: 12 to 19 + return cardLength >= 12 && cardLength <= 19; + } + if (inRange(b, "506099", "506198") + || inRange(b, "507865", "507964") + || inRange(b, "650002", "650027")) { + // Verve: 16, 18 or 19 + return cardLength == 16 || cardLength == 18 || cardLength == 19; + } + // Unknown scheme: accept any length. + return true; + } + + private static boolean inRange(String bin, String min, String max) { + int prefixLength = min.length(); + if (bin.length() < prefixLength) { + return false; + } + String prefix = bin.substring(0, prefixLength); + return prefix.compareTo(min) >= 0 && prefix.compareTo(max) <= 0; + } +} diff --git a/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidator.java b/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidator.java index 9526c33422c..f8c8ac5b5c9 100644 --- a/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidator.java +++ b/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidator.java @@ -19,7 +19,9 @@ import org.apache.hop.core.Const; import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.row.IValueMeta; import org.apache.hop.core.row.RowDataUtil; +import org.apache.hop.core.row.value.ValueMetaString; import org.apache.hop.core.util.Utils; import org.apache.hop.i18n.BaseMessages; import org.apache.hop.pipeline.Pipeline; @@ -91,6 +93,19 @@ public boolean processRow() throws HopException { } data.realCardTypeFieldname = resolve(meta.getCardType()); data.realNotValidMsgFieldname = resolve(meta.getNotValidMessage()); + data.outputFieldMetas.clear(); + if (meta.isUseBinDatabase()) { + for (BinOutputField outputField : meta.getOutputFields()) { + String realName = resolve(outputField.getName()); + if (!Utils.isEmpty(realName)) { + try { + data.outputFieldMetas.add(outputField.createValueMeta(realName)); + } catch (Exception e) { + throw new HopException(e); + } + } + } + } } // End If first Object[] outputRow = RowDataUtil.createResizedCopy(row, data.outputRowMeta.size()); @@ -101,7 +116,12 @@ public boolean processRow() throws HopException { fieldValue = Const.getDigitsOnly(fieldValue); } - ReturnIndicator rt = CreditCardVerifier.checkCC(fieldValue); + ReturnIndicator rt; + if (data.binDatabase != null) { + rt = CreditCardVerifier.checkCC(fieldValue, data.binDatabase); + } else { + rt = CreditCardVerifier.checkCC(fieldValue); + } // Check if Card is Valid? isValid = rt.CardValid; @@ -127,10 +147,21 @@ public boolean processRow() throws HopException { if (!Utils.isEmpty(data.realNotValidMsgFieldname)) { outputRow[rowIndex++] = unValid; } + // add extra output fields? + if (meta.isUseBinDatabase() && rt.extraValues != null) { + for (int i = 0; i < meta.getOutputFields().size(); i++) { + BinOutputField outputField = meta.getOutputFields().get(i); + String realName = resolve(outputField.getName()); + if (!Utils.isEmpty(realName)) { + String value = rt.extraValues.get(realName); + IValueMeta valueMeta = data.outputFieldMetas.get(i); + outputRow[rowIndex++] = convertValue(valueMeta, value); + } + } + } // add new values to the row. putRow(data.outputRowMeta, outputRow); // copy row to output rowset(s) - if (isRowLevel()) { logRowlevel( BaseMessages.getString( @@ -167,6 +198,14 @@ public boolean processRow() throws HopException { return true; } + private Object convertValue(IValueMeta valueMeta, String value) throws Exception { + if (value == null) { + return null; + } + return valueMeta.convertDataFromString( + value, new ValueMetaString(valueMeta.getName()), null, null, valueMeta.getTrimType()); + } + @Override public boolean init() { if (super.init()) { @@ -174,6 +213,37 @@ public boolean init() { logError(BaseMessages.getString(PKG, "CreditCardValidator.Error.ResultFieldMissing")); return false; } + if (meta.isUseBinDatabase()) { + String fileName = resolve(meta.getBinFileName()); + if (Utils.isEmpty(fileName)) { + logError(BaseMessages.getString(PKG, "CreditCardValidator.Error.BinFileNameMissing")); + return false; + } + try { + data.binDatabase = new BinDatabase(); + data.binDatabase.load( + this, + fileName, + resolve(meta.getBinCsvColumn()), + meta.getOutputFields(), + resolve(meta.getBinDelimiter()), + resolve(meta.getBinEnclosure()), + resolve(meta.getBinEncoding()), + meta.isBinHeaderPresent()); + if (data.binDatabase.getSkippedRows() > 0) { + logBasic( + BaseMessages.getString( + PKG, + "CreditCardValidator.Log.SkippedBinRows", + data.binDatabase.getSkippedRows())); + } + } catch (HopException e) { + logError( + BaseMessages.getString(PKG, "CreditCardValidator.Error.BinDatabaseLoad") + + e.getMessage()); + return false; + } + } return true; } return false; diff --git a/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidatorData.java b/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidatorData.java index 6a5d0d51d30..36daf1b4eb4 100644 --- a/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidatorData.java +++ b/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidatorData.java @@ -17,7 +17,10 @@ package org.apache.hop.pipeline.transforms.creditcardvalidator; +import java.util.ArrayList; +import java.util.List; import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; import org.apache.hop.pipeline.transform.BaseTransformData; import org.apache.hop.pipeline.transform.ITransformData; @@ -30,6 +33,8 @@ public class CreditCardValidatorData extends BaseTransformData implements ITrans public IRowMeta outputRowMeta; public int NrPrevFields; public IRowMeta previousRowMeta; + public BinDatabase binDatabase; + public List outputFieldMetas; public CreditCardValidatorData() { super(); @@ -37,5 +42,6 @@ public CreditCardValidatorData() { realResultFieldname = null; realCardTypeFieldname = null; realNotValidMsgFieldname = null; + outputFieldMetas = new ArrayList<>(); } } diff --git a/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidatorDialog.java b/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidatorDialog.java index 413a005cfa4..e29ab815763 100644 --- a/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidatorDialog.java +++ b/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidatorDialog.java @@ -17,25 +17,47 @@ package org.apache.hop.pipeline.transforms.creditcardvalidator; +import com.opencsv.CSVParserBuilder; +import com.opencsv.CSVReader; +import com.opencsv.CSVReaderBuilder; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; import org.apache.hop.core.Const; import org.apache.hop.core.exception.HopException; import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.value.ValueMetaBase; +import org.apache.hop.core.row.value.ValueMetaFactory; import org.apache.hop.core.util.Utils; import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.vfs.HopVfs; import org.apache.hop.i18n.BaseMessages; import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.ui.core.ConstUi; import org.apache.hop.ui.core.PropsUi; import org.apache.hop.ui.core.dialog.BaseDialog; import org.apache.hop.ui.core.dialog.ErrorDialog; +import org.apache.hop.ui.core.dialog.MessageBox; +import org.apache.hop.ui.core.widget.ColumnInfo; +import org.apache.hop.ui.core.widget.ComboVar; +import org.apache.hop.ui.core.widget.TableView; import org.apache.hop.ui.core.widget.TextVar; import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; import org.apache.hop.ui.pipeline.transform.ComponentSelectionListener; import org.eclipse.swt.SWT; import org.eclipse.swt.custom.CCombo; +import org.eclipse.swt.custom.CTabFolder; +import org.eclipse.swt.custom.CTabItem; import org.eclipse.swt.custom.ScrolledComposite; import org.eclipse.swt.events.FocusEvent; import org.eclipse.swt.events.FocusListener; import org.eclipse.swt.events.ModifyListener; +import org.eclipse.swt.events.SelectionAdapter; +import org.eclipse.swt.events.SelectionEvent; import org.eclipse.swt.graphics.Cursor; import org.eclipse.swt.graphics.Rectangle; import org.eclipse.swt.layout.FillLayout; @@ -47,6 +69,7 @@ import org.eclipse.swt.widgets.Group; import org.eclipse.swt.widgets.Label; import org.eclipse.swt.widgets.Shell; +import org.eclipse.swt.widgets.TableItem; public class CreditCardValidatorDialog extends BaseTransformDialog { private static final Class PKG = CreditCardValidatorMeta.class; @@ -62,6 +85,28 @@ public class CreditCardValidatorDialog extends BaseTransformDialog { private Button wgetOnlyDigits; + private Button wUseBinDatabase; + + private TextVar wBinFileName; + + private Button wbbBinFileName; + + private TextVar wBinDelimiter; + + private TextVar wBinEnclosure; + + private ComboVar wBinEncoding; + + private Button wBinHeaderPresent; + + private CCombo wBinCsvColumn; + + private Button wbGetColumns; + + private TableView wOutputFields; + + private List binCsvColumns = new ArrayList<>(); + private final CreditCardValidatorMeta input; public CreditCardValidatorDialog( @@ -82,14 +127,34 @@ public String open() { ModifyListener lsMod = e -> input.setChanged(); changed = input.hasChanged(); - ScrolledComposite sc = new ScrolledComposite(shell, SWT.V_SCROLL | SWT.H_SCROLL); + CTabFolder wTabFolder = new CTabFolder(shell, SWT.BORDER); + PropsUi.setLook(wTabFolder); + + addFieldsTab(wTabFolder, lsMod); + addBinDatabaseTab(wTabFolder, lsMod); + + FormData fdTabFolder = new FormData(); + fdTabFolder.left = new FormAttachment(0, 0); + fdTabFolder.top = new FormAttachment(wSpacer, margin); + fdTabFolder.right = new FormAttachment(100, 0); + fdTabFolder.bottom = new FormAttachment(wOk, -margin); + wTabFolder.setLayoutData(fdTabFolder); + + wTabFolder.setSelection(0); + + getData(); + focusTransformName(); + BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel()); + + return transformName; + } + + private void addFieldsTab(CTabFolder wTabFolder, ModifyListener lsMod) { + CTabItem wFieldsTab = new CTabItem(wTabFolder, SWT.NONE); + wFieldsTab.setText(BaseMessages.getString(PKG, "CreditCardValidatorDialog.Fields.Tab")); + + ScrolledComposite sc = new ScrolledComposite(wTabFolder, SWT.V_SCROLL | SWT.H_SCROLL); PropsUi.setLook(sc); - FormData fdSc = new FormData(); - fdSc.left = new FormAttachment(0, 0); - fdSc.top = new FormAttachment(wSpacer, 0); - fdSc.right = new FormAttachment(100, 0); - fdSc.bottom = new FormAttachment(wOk, margin); - sc.setLayoutData(fdSc); sc.setLayout(new FillLayout()); Composite wContent = new Composite(sc, SWT.NONE); @@ -245,21 +310,373 @@ public void focusGained(FocusEvent e) { sc.setMinWidth(bounds.width); sc.setMinHeight(bounds.height); - getData(); - focusTransformName(); - BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel()); + wFieldsTab.setControl(sc); + } - return transformName; + private void addBinDatabaseTab(CTabFolder wTabFolder, ModifyListener lsMod) { + CTabItem wBinTab = new CTabItem(wTabFolder, SWT.NONE); + wBinTab.setText(BaseMessages.getString(PKG, "CreditCardValidatorDialog.BinDatabase.Tab")); + + ScrolledComposite sc = new ScrolledComposite(wTabFolder, SWT.V_SCROLL | SWT.H_SCROLL); + PropsUi.setLook(sc); + sc.setLayout(new FillLayout()); + + Composite wContent = new Composite(sc, SWT.NONE); + PropsUi.setLook(wContent); + FormLayout contentLayout = new FormLayout(); + contentLayout.marginWidth = PropsUi.getFormMargin(); + contentLayout.marginHeight = PropsUi.getFormMargin(); + wContent.setLayout(contentLayout); + + Group wBinDatabase = new Group(wContent, SWT.SHADOW_NONE); + PropsUi.setLook(wBinDatabase); + wBinDatabase.setText( + BaseMessages.getString(PKG, "CreditCardValidatorDialog.BinDatabase.Label")); + + FormLayout binDatabaseGroupLayout = new FormLayout(); + binDatabaseGroupLayout.marginWidth = 10; + binDatabaseGroupLayout.marginHeight = 10; + wBinDatabase.setLayout(binDatabaseGroupLayout); + + // Use BIN database checkbox + Label wlUseBin = new Label(wBinDatabase, SWT.RIGHT); + wlUseBin.setText(BaseMessages.getString(PKG, "CreditCardValidatorDialog.UseBinDatabase.Label")); + PropsUi.setLook(wlUseBin); + FormData fdlUseBin = new FormData(); + fdlUseBin.left = new FormAttachment(0, 0); + fdlUseBin.right = new FormAttachment(middle, -margin); + fdlUseBin.top = new FormAttachment(0, margin); + wlUseBin.setLayoutData(fdlUseBin); + + wUseBinDatabase = new Button(wBinDatabase, SWT.CHECK); + PropsUi.setLook(wUseBinDatabase); + wUseBinDatabase.setToolTipText( + BaseMessages.getString(PKG, "CreditCardValidatorDialog.UseBinDatabase.Tooltip")); + FormData fdUseBin = new FormData(); + fdUseBin.left = new FormAttachment(middle, 0); + fdUseBin.top = new FormAttachment(0, margin); + wUseBinDatabase.setLayoutData(fdUseBin); + wUseBinDatabase.addSelectionListener( + new SelectionAdapter() { + @Override + public void widgetSelected(SelectionEvent e) { + input.setChanged(); + setBinDatabaseEnabled(); + } + }); + + // File name with browse + Label wlBinFileName = new Label(wBinDatabase, SWT.RIGHT); + wlBinFileName.setText( + BaseMessages.getString(PKG, "CreditCardValidatorDialog.BinFileName.Label")); + PropsUi.setLook(wlBinFileName); + FormData fdlBinFileName = new FormData(); + fdlBinFileName.left = new FormAttachment(0, 0); + fdlBinFileName.right = new FormAttachment(middle, -margin); + fdlBinFileName.top = new FormAttachment(wUseBinDatabase, margin); + wlBinFileName.setLayoutData(fdlBinFileName); + + wbbBinFileName = new Button(wBinDatabase, SWT.PUSH | SWT.CENTER); + PropsUi.setLook(wbbBinFileName); + wbbBinFileName.setText(BaseMessages.getString(PKG, "System.Button.Browse")); + wbbBinFileName.setToolTipText( + BaseMessages.getString(PKG, "System.Tooltip.BrowseForFileOrDirAndAdd")); + FormData fdbbBinFileName = new FormData(); + fdbbBinFileName.right = new FormAttachment(100, 0); + fdbbBinFileName.top = new FormAttachment(wUseBinDatabase, margin); + wbbBinFileName.setLayoutData(fdbbBinFileName); + wbbBinFileName.addListener( + SWT.Selection, + e -> + BaseDialog.presentFileDialog( + shell, + wBinFileName, + variables, + new String[] {"*.csv", "*.*"}, + new String[] { + BaseMessages.getString(PKG, "System.FileType.CSVFiles"), + BaseMessages.getString(PKG, "System.FileType.AllFiles") + }, + true)); + + wBinFileName = new TextVar(variables, wBinDatabase, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wBinFileName.setToolTipText( + BaseMessages.getString(PKG, "CreditCardValidatorDialog.BinFileName.Tooltip")); + PropsUi.setLook(wBinFileName); + wBinFileName.addModifyListener(lsMod); + FormData fdBinFileName = new FormData(); + fdBinFileName.left = new FormAttachment(middle, 0); + fdBinFileName.top = new FormAttachment(wUseBinDatabase, margin); + fdBinFileName.right = new FormAttachment(wbbBinFileName, -margin); + wBinFileName.setLayoutData(fdBinFileName); + + // Delimiter + Label wlBinDelimiter = new Label(wBinDatabase, SWT.RIGHT); + wlBinDelimiter.setText( + BaseMessages.getString(PKG, "CreditCardValidatorDialog.Delimiter.Label")); + PropsUi.setLook(wlBinDelimiter); + FormData fdlBinDelimiter = new FormData(); + fdlBinDelimiter.left = new FormAttachment(0, 0); + fdlBinDelimiter.right = new FormAttachment(middle, -margin); + fdlBinDelimiter.top = new FormAttachment(wBinFileName, margin); + wlBinDelimiter.setLayoutData(fdlBinDelimiter); + + wBinDelimiter = new TextVar(variables, wBinDatabase, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wBinDelimiter); + wBinDelimiter.addModifyListener(lsMod); + FormData fdBinDelimiter = new FormData(); + fdBinDelimiter.left = new FormAttachment(middle, 0); + fdBinDelimiter.top = new FormAttachment(wBinFileName, margin); + fdBinDelimiter.right = new FormAttachment(100, 0); + wBinDelimiter.setLayoutData(fdBinDelimiter); + + // Enclosure + Label wlBinEnclosure = new Label(wBinDatabase, SWT.RIGHT); + wlBinEnclosure.setText( + BaseMessages.getString(PKG, "CreditCardValidatorDialog.Enclosure.Label")); + PropsUi.setLook(wlBinEnclosure); + FormData fdlBinEnclosure = new FormData(); + fdlBinEnclosure.left = new FormAttachment(0, 0); + fdlBinEnclosure.right = new FormAttachment(middle, -margin); + fdlBinEnclosure.top = new FormAttachment(wBinDelimiter, margin); + wlBinEnclosure.setLayoutData(fdlBinEnclosure); + + wBinEnclosure = new TextVar(variables, wBinDatabase, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wBinEnclosure); + wBinEnclosure.addModifyListener(lsMod); + FormData fdBinEnclosure = new FormData(); + fdBinEnclosure.left = new FormAttachment(middle, 0); + fdBinEnclosure.top = new FormAttachment(wBinDelimiter, margin); + fdBinEnclosure.right = new FormAttachment(100, 0); + wBinEnclosure.setLayoutData(fdBinEnclosure); + + // Encoding + Label wlBinEncoding = new Label(wBinDatabase, SWT.RIGHT); + wlBinEncoding.setText(BaseMessages.getString(PKG, "CreditCardValidatorDialog.Encoding.Label")); + PropsUi.setLook(wlBinEncoding); + FormData fdlBinEncoding = new FormData(); + fdlBinEncoding.left = new FormAttachment(0, 0); + fdlBinEncoding.right = new FormAttachment(middle, -margin); + fdlBinEncoding.top = new FormAttachment(wBinEnclosure, margin); + wlBinEncoding.setLayoutData(fdlBinEncoding); + + wBinEncoding = new ComboVar(variables, wBinDatabase, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + PropsUi.setLook(wBinEncoding); + wBinEncoding.addModifyListener(lsMod); + FormData fdBinEncoding = new FormData(); + fdBinEncoding.left = new FormAttachment(middle, 0); + fdBinEncoding.top = new FormAttachment(wBinEnclosure, margin); + fdBinEncoding.right = new FormAttachment(100, 0); + wBinEncoding.setLayoutData(fdBinEncoding); + + // Header present + Label wlBinHeader = new Label(wBinDatabase, SWT.RIGHT); + wlBinHeader.setText( + BaseMessages.getString(PKG, "CreditCardValidatorDialog.HeaderPresent.Label")); + PropsUi.setLook(wlBinHeader); + FormData fdlBinHeader = new FormData(); + fdlBinHeader.left = new FormAttachment(0, 0); + fdlBinHeader.right = new FormAttachment(middle, -margin); + fdlBinHeader.top = new FormAttachment(wBinEncoding, margin); + wlBinHeader.setLayoutData(fdlBinHeader); + + wBinHeaderPresent = new Button(wBinDatabase, SWT.CHECK); + PropsUi.setLook(wBinHeaderPresent); + wBinHeaderPresent.setToolTipText( + BaseMessages.getString(PKG, "CreditCardValidatorDialog.HeaderPresent.Tooltip")); + FormData fdBinHeader = new FormData(); + fdBinHeader.left = new FormAttachment(middle, 0); + fdBinHeader.top = new FormAttachment(wBinEncoding, margin); + wBinHeaderPresent.setLayoutData(fdBinHeader); + wBinHeaderPresent.addSelectionListener(new ComponentSelectionListener(input)); + wBinHeaderPresent.addSelectionListener( + new SelectionAdapter() { + @Override + public void widgetSelected(SelectionEvent e) { + if (!wBinHeaderPresent.getSelection() && hasHeaderMappings()) { + MessageBox mb = new MessageBox(shell, SWT.YES | SWT.NO | SWT.ICON_WARNING); + mb.setText( + BaseMessages.getString( + PKG, "CreditCardValidatorDialog.HeaderDisabledWarning.Title")); + mb.setMessage( + BaseMessages.getString( + PKG, "CreditCardValidatorDialog.HeaderDisabledWarning.Message")); + if (mb.open() == SWT.NO) { + wBinHeaderPresent.setSelection(true); + } else { + clearHeaderMappings(); + } + } + setBinDatabaseEnabled(); + } + }); + + // BIN CSV column + Label wlBinCsvColumn = new Label(wBinDatabase, SWT.RIGHT); + wlBinCsvColumn.setText( + BaseMessages.getString(PKG, "CreditCardValidatorDialog.BinCsvColumn.Label")); + PropsUi.setLook(wlBinCsvColumn); + FormData fdlBinCsvColumn = new FormData(); + fdlBinCsvColumn.left = new FormAttachment(0, 0); + fdlBinCsvColumn.right = new FormAttachment(middle, -margin); + fdlBinCsvColumn.top = new FormAttachment(wBinHeaderPresent, margin); + wlBinCsvColumn.setLayoutData(fdlBinCsvColumn); + + wBinCsvColumn = new CCombo(wBinDatabase, SWT.BORDER); + wBinCsvColumn.setEditable(true); + PropsUi.setLook(wBinCsvColumn); + wBinCsvColumn.addModifyListener(lsMod); + FormData fdBinCsvColumn = new FormData(); + fdBinCsvColumn.left = new FormAttachment(middle, 0); + fdBinCsvColumn.top = new FormAttachment(wBinHeaderPresent, margin); + fdBinCsvColumn.right = new FormAttachment(100, 0); + wBinCsvColumn.setLayoutData(fdBinCsvColumn); + wBinCsvColumn.addFocusListener( + new FocusListener() { + @Override + public void focusLost(FocusEvent e) { + // Disable focuslost + } + + @Override + public void focusGained(FocusEvent e) { + loadBinCsvColumns(); + } + }); + + // Get fields button + wbGetColumns = new Button(wBinDatabase, SWT.PUSH | SWT.CENTER); + PropsUi.setLook(wbGetColumns); + wbGetColumns.setText( + BaseMessages.getString(PKG, "CreditCardValidatorDialog.GetColumns.Button")); + wbGetColumns.setToolTipText( + BaseMessages.getString(PKG, "CreditCardValidatorDialog.GetColumns.Tooltip")); + FormData fdbGetColumns = new FormData(); + fdbGetColumns.right = new FormAttachment(100, 0); + fdbGetColumns.top = new FormAttachment(wBinCsvColumn, margin); + wbGetColumns.setLayoutData(fdbGetColumns); + wbGetColumns.addSelectionListener( + new SelectionAdapter() { + @Override + public void widgetSelected(SelectionEvent e) { + getFields(); + } + }); + + // Extra output fields + Label wlOutputFields = new Label(wBinDatabase, SWT.RIGHT); + wlOutputFields.setText( + BaseMessages.getString(PKG, "CreditCardValidatorDialog.OutputFieldList.Label")); + PropsUi.setLook(wlOutputFields); + FormData fdlOutputFields = new FormData(); + fdlOutputFields.left = new FormAttachment(0, 0); + fdlOutputFields.right = new FormAttachment(middle, -margin); + fdlOutputFields.top = new FormAttachment(wBinCsvColumn, margin); + wlOutputFields.setLayoutData(fdlOutputFields); + + ColumnInfo[] outputColInfos = + new ColumnInfo[] { + new ColumnInfo( + BaseMessages.getString(PKG, "CreditCardValidatorDialog.OutputName.Column"), + ColumnInfo.COLUMN_TYPE_TEXT, + false, + false, + 120), + new ColumnInfo( + BaseMessages.getString(PKG, "CreditCardValidatorDialog.OutputColumn.Column"), + ColumnInfo.COLUMN_TYPE_CCOMBO, + new String[0], + false), + new ColumnInfo( + BaseMessages.getString(PKG, "CreditCardValidatorDialog.OutputType.Column"), + ColumnInfo.COLUMN_TYPE_CCOMBO, + ValueMetaFactory.getValueMetaNames(), + true), + new ColumnInfo( + BaseMessages.getString(PKG, "CreditCardValidatorDialog.OutputFormat.Column"), + ColumnInfo.COLUMN_TYPE_FORMAT, + 2), + new ColumnInfo( + BaseMessages.getString(PKG, "CreditCardValidatorDialog.OutputLength.Column"), + ColumnInfo.COLUMN_TYPE_TEXT, + false), + new ColumnInfo( + BaseMessages.getString(PKG, "CreditCardValidatorDialog.OutputPrecision.Column"), + ColumnInfo.COLUMN_TYPE_TEXT, + false), + new ColumnInfo( + BaseMessages.getString(PKG, "CreditCardValidatorDialog.OutputCurrency.Column"), + ColumnInfo.COLUMN_TYPE_TEXT, + false), + new ColumnInfo( + BaseMessages.getString(PKG, "CreditCardValidatorDialog.OutputDecimal.Column"), + ColumnInfo.COLUMN_TYPE_TEXT, + false), + new ColumnInfo( + BaseMessages.getString(PKG, "CreditCardValidatorDialog.OutputGroup.Column"), + ColumnInfo.COLUMN_TYPE_TEXT, + false), + new ColumnInfo( + BaseMessages.getString(PKG, "CreditCardValidatorDialog.OutputTrim.Column"), + ColumnInfo.COLUMN_TYPE_CCOMBO, + ValueMetaBase.trimTypeDesc, + true), + }; + outputColInfos[1].setComboValueSupplier(() -> binCsvColumns.toArray(new String[0])); + wOutputFields = + new TableView( + variables, + wBinDatabase, + SWT.FULL_SELECTION | SWT.MULTI | SWT.BORDER, + outputColInfos, + input.getOutputFields().size(), + lsMod, + props); + PropsUi.setLook(wOutputFields); + FormData fdOutputFields = new FormData(); + fdOutputFields.left = new FormAttachment(middle, 0); + fdOutputFields.top = new FormAttachment(wBinCsvColumn, margin); + fdOutputFields.right = new FormAttachment(wbGetColumns, -margin); + fdOutputFields.bottom = new FormAttachment(100, -margin); + wOutputFields.setLayoutData(fdOutputFields); + + FormData fdBinDatabase = new FormData(); + fdBinDatabase.left = new FormAttachment(0, margin); + fdBinDatabase.top = new FormAttachment(0, margin); + fdBinDatabase.right = new FormAttachment(100, -margin); + fdBinDatabase.bottom = new FormAttachment(100, -margin); + wBinDatabase.setLayoutData(fdBinDatabase); + + wContent.pack(); + Rectangle bounds = wContent.getBounds(); + sc.setContent(wContent); + sc.setExpandHorizontal(true); + sc.setExpandVertical(true); + sc.setMinWidth(bounds.width); + sc.setMinHeight(bounds.height); + + wBinTab.setControl(sc); } /** Copy information from the meta-data input to the dialog fields. */ public void getData() { - wFieldName.setText(Const.NVL(input.getFieldName(), "")); wgetOnlyDigits.setSelection(input.isOnlyDigits()); wResult.setText(Const.NVL(input.getResultFieldName(), "")); wFileType.setText(Const.NVL(input.getCardType(), "")); wNotValidMsg.setText(Const.NVL(input.getNotValidMessage(), "")); + wUseBinDatabase.setSelection(input.isUseBinDatabase()); + wBinFileName.setText(Const.NVL(input.getBinFileName(), "")); + wBinDelimiter.setText(Const.NVL(input.getBinDelimiter(), ",")); + wBinEnclosure.setText(Const.NVL(input.getBinEnclosure(), "\"")); + String encoding = Const.NVL(input.getBinEncoding(), "UTF-8"); + wBinEncoding.setItems(ConstUi.getEncodings()); + wBinEncoding.setText(encoding); + wBinHeaderPresent.setSelection(input.isBinHeaderPresent()); + wBinCsvColumn.setText(Const.NVL(input.getBinCsvColumn(), "")); + populateOutputFieldsTable(); + setBinDatabaseEnabled(); + loadBinCsvColumns(); } private void cancel() { @@ -277,12 +694,191 @@ private void ok() { input.setResultFieldName(wResult.getText()); input.setCardType(wFileType.getText()); input.setNotValidMessage(wNotValidMsg.getText()); + input.setUseBinDatabase(wUseBinDatabase.getSelection()); + input.setBinFileName(wBinFileName.getText()); + input.setBinDelimiter(wBinDelimiter.getText()); + input.setBinEnclosure(wBinEnclosure.getText()); + input.setBinEncoding(wBinEncoding.getText()); + input.setBinHeaderPresent(wBinHeaderPresent.getSelection()); + input.setBinCsvColumn(wBinCsvColumn.getText()); + readOutputFieldsTable(); // return value transformName = wTransformName.getText(); dispose(); } + private void populateOutputFieldsTable() { + wOutputFields.clearAll(); + for (BinOutputField outputField : input.getOutputFields()) { + wOutputFields.add( + Const.NVL(outputField.getName(), ""), + Const.NVL(outputField.getColumn(), ""), + ValueMetaFactory.getValueMetaName(outputField.getType()), + Const.NVL(outputField.getFormat(), ""), + Integer.toString(outputField.getLength()), + Integer.toString(outputField.getPrecision()), + Const.NVL(outputField.getCurrency(), ""), + Const.NVL(outputField.getDecimal(), ""), + Const.NVL(outputField.getGroup(), ""), + ValueMetaBase.getTrimTypeDesc(outputField.getTrimType())); + } + wOutputFields.removeEmptyRows(); + wOutputFields.setRowNums(); + wOutputFields.optWidth(true); + } + + private void readOutputFieldsTable() { + List outputFields = new ArrayList<>(); + int rows = wOutputFields.nrNonEmpty(); + for (int i = 0; i < rows; i++) { + TableItem item = wOutputFields.getNonEmpty(i); + String name = item.getText(1); + String column = item.getText(2); + if (Utils.isEmpty(name) && Utils.isEmpty(column)) { + continue; + } + BinOutputField outputField = new BinOutputField(name, column); + outputField.setType(ValueMetaFactory.getIdForValueMeta(item.getText(3))); + outputField.setFormat(item.getText(4)); + outputField.setLength(Const.toInt(item.getText(5), -1)); + outputField.setPrecision(Const.toInt(item.getText(6), -1)); + outputField.setCurrency(item.getText(7)); + outputField.setDecimal(item.getText(8)); + outputField.setGroup(item.getText(9)); + outputField.setTrimType(ValueMetaBase.getTrimTypeByDesc(item.getText(10))); + outputFields.add(outputField); + } + input.setOutputFields(outputFields); + } + + private void setBinDatabaseEnabled() { + boolean enabled = wUseBinDatabase.getSelection(); + boolean hasHeader = enabled && wBinHeaderPresent.getSelection(); + wBinFileName.setEnabled(enabled); + wbbBinFileName.setEnabled(enabled); + wBinDelimiter.setEnabled(enabled); + wBinEnclosure.setEnabled(enabled); + wBinEncoding.setEnabled(enabled); + wBinHeaderPresent.setEnabled(enabled); + wBinCsvColumn.setEnabled(hasHeader); + wbGetColumns.setEnabled(hasHeader); + wOutputFields.setEnabled(enabled); + } + + private boolean hasHeaderMappings() { + if (!Utils.isEmpty(wBinCsvColumn.getText())) { + return true; + } + int rows = wOutputFields.nrNonEmpty(); + for (int i = 0; i < rows; i++) { + TableItem item = wOutputFields.getNonEmpty(i); + if (!Utils.isEmpty(item.getText(2))) { + return true; + } + } + return false; + } + + private void clearHeaderMappings() { + wBinCsvColumn.setText(""); + int rows = wOutputFields.nrNonEmpty(); + for (int i = 0; i < rows; i++) { + TableItem item = wOutputFields.getNonEmpty(i); + item.setText(2, ""); + } + wOutputFields.removeEmptyRows(); + wOutputFields.setRowNums(); + wOutputFields.optWidth(true); + } + + private void loadBinCsvColumns() { + if (!wBinHeaderPresent.getSelection()) { + return; + } + readHeader( + header -> { + binCsvColumns.clear(); + for (String column : header) { + binCsvColumns.add(column); + } + String current = wBinCsvColumn.getText(); + wBinCsvColumn.removeAll(); + wBinCsvColumn.setItems(binCsvColumns.toArray(new String[0])); + wBinCsvColumn.setText(current); + }); + } + + private void getFields() { + if (!wBinHeaderPresent.getSelection()) { + return; + } + readHeader( + header -> { + binCsvColumns.clear(); + for (String column : header) { + binCsvColumns.add(column); + } + String current = wBinCsvColumn.getText(); + wBinCsvColumn.removeAll(); + wBinCsvColumn.setItems(binCsvColumns.toArray(new String[0])); + wBinCsvColumn.setText(current); + + for (String column : header) { + wOutputFields.add( + column, + column, + ValueMetaFactory.getValueMetaName(IValueMeta.TYPE_STRING), + "", + "-1", + "-1", + "", + "", + "", + ValueMetaBase.getTrimTypeDesc(IValueMeta.TRIM_TYPE_NONE)); + } + wOutputFields.removeEmptyRows(); + wOutputFields.setRowNums(); + wOutputFields.optWidth(true); + }); + } + + private void readHeader(java.util.function.Consumer consumer) { + String fileName = variables.resolve(wBinFileName.getText()); + if (Utils.isEmpty(fileName)) { + return; + } + String delimiter = variables.resolve(wBinDelimiter.getText()); + String enclosure = variables.resolve(wBinEnclosure.getText()); + String encoding = variables.resolve(wBinEncoding.getText()); + Charset charset; + try { + charset = Charset.forName(Utils.isEmpty(encoding) ? "UTF-8" : encoding); + } catch (Exception e) { + charset = StandardCharsets.UTF_8; + } + try (InputStream in = HopVfs.getInputStream(fileName, variables); + CSVReader reader = + new CSVReaderBuilder(new InputStreamReader(in, charset)) + .withCSVParser( + new CSVParserBuilder() + .withSeparator(Utils.isEmpty(delimiter) ? ',' : delimiter.charAt(0)) + .withQuoteChar(Utils.isEmpty(enclosure) ? '"' : enclosure.charAt(0)) + .build()) + .build()) { + String[] header = reader.readNext(); + if (header != null) { + consumer.accept(header); + } + } catch (Exception e) { + new ErrorDialog( + shell, + BaseMessages.getString(PKG, "CreditCardValidatorDialog.GetColumns.DialogTitle"), + BaseMessages.getString(PKG, "CreditCardValidatorDialog.GetColumns.ErrorMessage"), + e); + } + } + private void get() { if (!gotPreviousFields) { try { diff --git a/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidatorMeta.java b/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidatorMeta.java index e0e81e84b90..35ce31d09c7 100644 --- a/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidatorMeta.java +++ b/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidatorMeta.java @@ -17,10 +17,12 @@ package org.apache.hop.pipeline.transforms.creditcardvalidator; +import java.util.ArrayList; import java.util.List; import org.apache.hop.core.CheckResult; import org.apache.hop.core.ICheckResult; import org.apache.hop.core.annotations.Transform; +import org.apache.hop.core.exception.HopPluginException; import org.apache.hop.core.exception.HopTransformException; import org.apache.hop.core.row.IRowMeta; import org.apache.hop.core.row.IValueMeta; @@ -65,8 +67,33 @@ public class CreditCardValidatorMeta @HopMetadataProperty(key = "onlydigits") private boolean onlyDigits; + @HopMetadataProperty(key = "usebindatabase") + private boolean useBinDatabase; + + @HopMetadataProperty(key = "binfilename") + private String binFileName; + + @HopMetadataProperty(key = "bindelimiter") + private String binDelimiter; + + @HopMetadataProperty(key = "binenclosure") + private String binEnclosure; + + @HopMetadataProperty(key = "binencoding") + private String binEncoding; + + @HopMetadataProperty(key = "binheaderpresent") + private boolean binHeaderPresent; + + @HopMetadataProperty(key = "bincsvcolumn") + private String binCsvColumn; + + @HopMetadataProperty(key = "outputfield", groupKey = "outputfields") + private List outputFields; + public CreditCardValidatorMeta() { super(); // allocate BaseTransformMeta + outputFields = new ArrayList<>(); } @Override @@ -75,6 +102,14 @@ public void setDefault() { onlyDigits = false; cardType = "card type"; notValidMessage = "not valid message"; + useBinDatabase = false; + binFileName = ""; + binDelimiter = ","; + binEnclosure = "\""; + binEncoding = "UTF-8"; + binHeaderPresent = true; + binCsvColumn = ""; + outputFields = new ArrayList<>(); } @Override @@ -104,6 +139,19 @@ public void getFields( v.setOrigin(name); inputRowMeta.addValueMeta(v); } + for (BinOutputField outputField : outputFields) { + String realName = variables.resolve(outputField.getName()); + if (!Utils.isEmpty(realName) && useBinDatabase) { + IValueMeta v; + try { + v = outputField.createValueMeta(realName); + } catch (HopPluginException e) { + throw new HopTransformException(e); + } + v.setOrigin(name); + inputRowMeta.addValueMeta(v); + } + } } @Override @@ -255,4 +303,148 @@ public boolean isOnlyDigits() { public void setOnlyDigits(boolean onlyDigits) { this.onlyDigits = onlyDigits; } + + /** + * Gets useBinDatabase + * + * @return value of useBinDatabase + */ + public boolean isUseBinDatabase() { + return useBinDatabase; + } + + /** + * Sets useBinDatabase + * + * @param useBinDatabase value of useBinDatabase + */ + public void setUseBinDatabase(boolean useBinDatabase) { + this.useBinDatabase = useBinDatabase; + } + + /** + * Gets binFileName + * + * @return value of binFileName + */ + public String getBinFileName() { + return binFileName; + } + + /** + * Sets binFileName + * + * @param binFileName value of binFileName + */ + public void setBinFileName(String binFileName) { + this.binFileName = binFileName; + } + + /** + * Gets binDelimiter + * + * @return value of binDelimiter + */ + public String getBinDelimiter() { + return binDelimiter; + } + + /** + * Sets binDelimiter + * + * @param binDelimiter value of binDelimiter + */ + public void setBinDelimiter(String binDelimiter) { + this.binDelimiter = binDelimiter; + } + + /** + * Gets binEnclosure + * + * @return value of binEnclosure + */ + public String getBinEnclosure() { + return binEnclosure; + } + + /** + * Sets binEnclosure + * + * @param binEnclosure value of binEnclosure + */ + public void setBinEnclosure(String binEnclosure) { + this.binEnclosure = binEnclosure; + } + + /** + * Gets binEncoding + * + * @return value of binEncoding + */ + public String getBinEncoding() { + return binEncoding; + } + + /** + * Sets binEncoding + * + * @param binEncoding value of binEncoding + */ + public void setBinEncoding(String binEncoding) { + this.binEncoding = binEncoding; + } + + /** + * Gets binHeaderPresent + * + * @return value of binHeaderPresent + */ + public boolean isBinHeaderPresent() { + return binHeaderPresent; + } + + /** + * Sets binHeaderPresent + * + * @param binHeaderPresent value of binHeaderPresent + */ + public void setBinHeaderPresent(boolean binHeaderPresent) { + this.binHeaderPresent = binHeaderPresent; + } + + /** + * Gets binCsvColumn + * + * @return value of binCsvColumn + */ + public String getBinCsvColumn() { + return binCsvColumn; + } + + /** + * Sets binCsvColumn + * + * @param binCsvColumn value of binCsvColumn + */ + public void setBinCsvColumn(String binCsvColumn) { + this.binCsvColumn = binCsvColumn; + } + + /** + * Gets outputFields + * + * @return value of outputFields + */ + public List getOutputFields() { + return outputFields; + } + + /** + * Sets outputFields + * + * @param outputFields value of outputFields + */ + public void setOutputFields(List outputFields) { + this.outputFields = outputFields; + } } diff --git a/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardVerifier.java b/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardVerifier.java index 1db27904d36..dde67840fcc 100644 --- a/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardVerifier.java +++ b/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardVerifier.java @@ -19,6 +19,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.hop.core.Const; import org.apache.hop.core.util.Utils; import org.apache.hop.i18n.BaseMessages; @@ -119,6 +120,56 @@ public static ReturnIndicator checkCC(String cardNumber) { return ri; } + /** Validate a card number against an external BIN database. */ + public static ReturnIndicator checkCC(String cardNumber, BinDatabase binDatabase) { + ReturnIndicator ri = new ReturnIndicator(); + + if (Utils.isEmpty(cardNumber)) { + ri.UnValidMsg = BaseMessages.getString(PKG, "CreditCardValidator.Log.EmptyNumber"); + return ri; + } + + Matcher m = Pattern.compile("[^\\d\\s.-]").matcher(cardNumber); + if (m.find()) { + ri.UnValidMsg = BaseMessages.getString(PKG, "CreditCardValidator.OnlyNumbers"); + return ri; + } + + String digits = Const.getDigitsOnly(cardNumber); + if (binDatabase == null) { + if (luhnValidate(digits)) { + ri.CardValid = true; + } else { + ri.UnValidMsg = BaseMessages.getString(PKG, "CreditCardValidator.Log.CardNotValid"); + } + return ri; + } + + BinDatabase.BinRecord record = binDatabase.lookup(digits); + if (record == null) { + ri.UnValidMsg = BaseMessages.getString(PKG, "CreditCardValidator.Log.CardNotValid"); + return ri; + } + + if (!CardScheme.isValidLength(digits, digits.length())) { + ri.UnValidMsg = BaseMessages.getString(PKG, "CreditCardValidator.Log.CardNotValid"); + return ri; + } + + if (luhnValidate(digits)) { + ri.CardValid = true; + ri.CardType = record.getCardType(); + ri.extraValues = record.getExtraValues(); + } else { + ri.CardValid = false; + ri.UnValidMsg = + BaseMessages.getString( + PKG, "CreditCardValidator.Log.NotValidCardType", record.getCardType()); + } + + return ri; + } + // The Luhn algorithm is basically a CRC type // system for checking the validity of an entry. // All major credit cards use numbers that will diff --git a/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/ReturnIndicator.java b/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/ReturnIndicator.java index ee0c778c3d9..c444892595f 100644 --- a/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/ReturnIndicator.java +++ b/plugins/transforms/creditcardvalidator/src/main/java/org/apache/hop/pipeline/transforms/creditcardvalidator/ReturnIndicator.java @@ -17,15 +17,20 @@ package org.apache.hop.pipeline.transforms.creditcardvalidator; +import java.util.HashMap; +import java.util.Map; + public class ReturnIndicator { public String CardType; public boolean CardValid; public String UnValidMsg; + public Map extraValues; public ReturnIndicator() { super(); CardValid = false; CardType = null; UnValidMsg = null; + extraValues = new HashMap<>(); } } diff --git a/plugins/transforms/creditcardvalidator/src/main/resources/org/apache/hop/pipeline/transforms/creditcardvalidator/messages/messages_en_US.properties b/plugins/transforms/creditcardvalidator/src/main/resources/org/apache/hop/pipeline/transforms/creditcardvalidator/messages/messages_en_US.properties index cfff7c560b9..bc30c6c6866 100644 --- a/plugins/transforms/creditcardvalidator/src/main/resources/org/apache/hop/pipeline/transforms/creditcardvalidator/messages/messages_en_US.properties +++ b/plugins/transforms/creditcardvalidator/src/main/resources/org/apache/hop/pipeline/transforms/creditcardvalidator/messages/messages_en_US.properties @@ -18,6 +18,11 @@ CreditCardValidator.Description=The Credit card validator transform will help you tell\:\n(1) if a credit card number is valid (uses LUHN10 (MOD-10) algorithm)\n(2) which credit card vendor handles that number\n(VISA, MasterCard, Diners Club, EnRoute, American Express (AMEX),...) CreditCardValidator.Error.CardFieldMissing=Card number fieldname is missing\! CreditCardValidator.Error.ResultFieldMissing=Result fieldname is missing\! +CreditCardValidator.Error.BinFileNameMissing=The BIN database file location is missing\! +CreditCardValidator.Error.BinDatabaseEmpty=The BIN database is empty\! +CreditCardValidator.Error.BinDatabaseRead=Unable to read the BIN database\! +CreditCardValidator.Error.BinDatabaseLoad=Unable to load the BIN database\: +CreditCardValidator.Error.BinColumnNotFound=BIN column ''{0}'' was not found in the dataset header\! CreditCardValidator.ErrorInTransformRunning=Because of an error, this transform can''t continue\: CreditCardValidator.Exception.CouldnotFindField=Couldn''t find field ''{0}'' in row\! CreditCardValidator.Exception.ResultFieldMissing=Result fieldname is missing\! @@ -25,6 +30,8 @@ CreditCardValidator.getOnlyDigits.Label=Get only digits CreditCardValidator.getOnlyDigits.Tooltip=Extract only digits from credit card before checking. CreditCardValidator.LineNumber=linenr {0} CreditCardValidator.Log.CardNotValid=This card is not valid\! +CreditCardValidator.Log.NotValidCardType=Not a valid number for {0}. +CreditCardValidator.Log.SkippedBinRows=Skipped {0} BIN row(s) (empty, longer than 8 digits, or non-numeric). CreditCardValidator.Log.EmptyNumber=Card number cannot be blank\! CreditCardValidator.Log.NotValidAirplus=Not a valid AIRPLUS CreditCardValidator.Log.NotValidAmericanExpress=Not a valid number for American Express. @@ -50,10 +57,50 @@ CreditCardValidatorDialog.FailedToGetFields.DialogTitle=Get fields failed CreditCardValidatorDialog.FieldName.Label=Credit Card field CreditCardValidatorDialog.NotValidMsg.Label=Not valid Message CreditCardValidatorDialog.NotValidMsg.Tooltip=Not valid Message +CreditCardValidatorDialog.Issuer.Label=Issuer field +CreditCardValidatorDialog.Issuer.Tooltip=Optional output field for the card issuer from the BIN database.\nLeave empty to omit. +CreditCardValidatorDialog.Country.Label=Country field +CreditCardValidatorDialog.Country.Tooltip=Optional output field for the issuer country from the BIN database.\nLeave empty to omit. CreditCardValidatorDialog.OutputFields.Label=Output fields CreditCardValidatorDialog.ResultField.Label=Result fieldname CreditCardValidatorDialog.ResultField.Tooltip=Specify here the name of the result fieldname (Boolean)\nadded in the input stream CreditCardValidatorDialog.Shell.Title=Credit card validator +CreditCardValidatorDialog.Fields.Tab=Fields +CreditCardValidatorDialog.BinDatabase.Tab=BIN Files +CreditCardValidatorDialog.BinDatabase.Label=BIN Files +CreditCardValidatorDialog.UseBinDatabase.Label=Use BIN database +CreditCardValidatorDialog.UseBinDatabase.Tooltip=When enabled, validate the card against an external CSV BIN database.\nWhen disabled, the built-in card type detection is used. +CreditCardValidatorDialog.BinFileName.Label=BIN database file +CreditCardValidatorDialog.BinFileName.Tooltip=Specify the CSV file containing the BIN database.\nBIN values are limited to 8 digits. +CreditCardValidatorDialog.Delimiter.Label=Delimiter +CreditCardValidatorDialog.Delimiter.Tooltip=The delimiter character used in the CSV file. +CreditCardValidatorDialog.Enclosure.Label=Enclosure +CreditCardValidatorDialog.Enclosure.Tooltip=The enclosure (quote) character used in the CSV file. +CreditCardValidatorDialog.Encoding.Label=Encoding +CreditCardValidatorDialog.Encoding.Tooltip=The character encoding of the CSV file. +CreditCardValidatorDialog.HeaderPresent.Label=Header row present +CreditCardValidatorDialog.HeaderPresent.Tooltip=When enabled, the first row of the CSV is treated as the header.\nWhen disabled, columns are mapped by fixed position. +CreditCardValidatorDialog.HeaderDisabledWarning.Title=Disable header row +CreditCardValidatorDialog.HeaderDisabledWarning.Message=Header-based column mappings (BIN CSV Column and extra output CSV columns) will be ignored and mapped by fixed position.\n\nContinue and clear these mappings? +CreditCardValidatorDialog.GetColumns.Button=Get columns +CreditCardValidatorDialog.GetColumns.Tooltip=Read the CSV header columns from the BIN database file. +CreditCardValidatorDialog.GetColumns.DialogTitle=Select column +CreditCardValidatorDialog.GetColumns.DialogMessage=Select a CSV column to add to the mapping. +CreditCardValidatorDialog.GetColumns.NoFileMessage=Please specify the BIN database file first\! +CreditCardValidatorDialog.GetColumns.ErrorMessage=Unable to read the CSV header\! +CreditCardValidatorDialog.BinCsvColumn.Label=BIN CSV Column +CreditCardValidatorDialog.BinCsvColumn.Tooltip=Select the CSV header column that contains the BIN values. +CreditCardValidatorDialog.OutputFieldList.Label=Extra output fields +CreditCardValidatorDialog.OutputName.Column=Field name +CreditCardValidatorDialog.OutputColumn.Column=CSV column +CreditCardValidatorDialog.OutputType.Column=Type +CreditCardValidatorDialog.OutputFormat.Column=Format +CreditCardValidatorDialog.OutputLength.Column=Length +CreditCardValidatorDialog.OutputPrecision.Column=Precision +CreditCardValidatorDialog.OutputCurrency.Column=Currency +CreditCardValidatorDialog.OutputDecimal.Column=Decimal +CreditCardValidatorDialog.OutputGroup.Column=Group +CreditCardValidatorDialog.OutputTrim.Column=Trim CreditCardValidatorDialog.TransformName.Label=Transform name CreditCardValidatorMeta.CheckResult.CardFieldMissing=Credit Card number field name is missing\! CreditCardValidatorMeta.CheckResult.CardFieldOK=Credit Card number field name was specified diff --git a/plugins/transforms/creditcardvalidator/src/test/java/org/apache/hop/pipeline/transforms/creditcardvalidator/BinDatabaseTest.java b/plugins/transforms/creditcardvalidator/src/test/java/org/apache/hop/pipeline/transforms/creditcardvalidator/BinDatabaseTest.java new file mode 100644 index 00000000000..28857874454 --- /dev/null +++ b/plugins/transforms/creditcardvalidator/src/test/java/org/apache/hop/pipeline/transforms/creditcardvalidator/BinDatabaseTest.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.creditcardvalidator; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import org.apache.hop.core.variables.Variables; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class BinDatabaseTest { + + @Test + void testLoadWithHeader(@TempDir Path tempDir) throws Exception { + Path file = tempDir.resolve("bins.csv"); + try (OutputStream out = Files.newOutputStream(file)) { + out.write( + ("BIN,CARDTYPE,ISSUERS,COUNTRY\n" + + "512003,MASTERCARD,PT. Bank Central Asia,INDONESIA\n" + + "51,MASTERCARD,Generic,\n") + .getBytes(StandardCharsets.UTF_8)); + } + + BinDatabase db = new BinDatabase(); + db.load( + new Variables(), + file.toUri().toString(), + "BIN", + new ArrayList<>(), + ",", + "\"", + "UTF-8", + true); + + assertEquals(2, db.size()); + BinDatabase.BinRecord record = db.lookup("5120031234567890"); + assertNotNull(record); + assertEquals("MASTERCARD", record.getCardType()); + } + + @Test + void testLongestPrefixMatch(@TempDir Path tempDir) throws Exception { + Path file = tempDir.resolve("bins.csv"); + try (OutputStream out = Files.newOutputStream(file)) { + out.write( + ("BIN,CARDTYPE,ISSUERS,COUNTRY\n" + + "51,MASTERCARD,Generic,\n" + + "512003,MASTERCARD,BCA,INDONESIA\n") + .getBytes(StandardCharsets.UTF_8)); + } + + BinDatabase db = new BinDatabase(); + db.load( + new Variables(), file.toUri().toString(), "", new ArrayList<>(), ",", "\"", "UTF-8", true); + + BinDatabase.BinRecord record = db.lookup("5120031234567890"); + assertNotNull(record); + assertEquals("MASTERCARD", record.getCardType()); + } + + @Test + void testLoadWithoutHeaderUsesPositionalOrder(@TempDir Path tempDir) throws Exception { + Path file = tempDir.resolve("bins.csv"); + try (OutputStream out = Files.newOutputStream(file)) { + out.write(("512003,MASTERCARD,BCA,INDONESIA\n").getBytes(StandardCharsets.UTF_8)); + } + + BinDatabase db = new BinDatabase(); + db.load( + new Variables(), file.toUri().toString(), "", new ArrayList<>(), ",", "\"", "UTF-8", false); + + assertEquals(1, db.size()); + BinDatabase.BinRecord record = db.lookup("5120031234567890"); + assertNotNull(record); + assertEquals("MASTERCARD", record.getCardType()); + } + + @Test + void testExtraOutputField(@TempDir Path tempDir) throws Exception { + Path file = tempDir.resolve("bins.csv"); + try (OutputStream out = Files.newOutputStream(file)) { + out.write( + ("BIN,CARDTYPE,ISSUERS,COUNTRY\n" + "512003,MASTERCARD,BCA,INDONESIA\n") + .getBytes(StandardCharsets.UTF_8)); + } + + List outputFields = new ArrayList<>(); + outputFields.add(new BinOutputField("bank_name", "ISSUERS")); + + BinDatabase db = new BinDatabase(); + db.load(new Variables(), file.toUri().toString(), "", outputFields, ",", "\"", "UTF-8", true); + + BinDatabase.BinRecord record = db.lookup("5120031234567890"); + assertNotNull(record); + assertEquals("BCA", record.getValue("bank_name")); + } + + @Test + void testExtraOutputFieldWithoutHeaderMapsPositionally(@TempDir Path tempDir) throws Exception { + Path file = tempDir.resolve("bins.csv"); + try (OutputStream out = Files.newOutputStream(file)) { + out.write(("512003,MASTERCARD,BCA,INDONESIA,BANKVALUE\n").getBytes(StandardCharsets.UTF_8)); + } + + List outputFields = new ArrayList<>(); + outputFields.add(new BinOutputField("bank_name", "")); + + BinDatabase db = new BinDatabase(); + db.load(new Variables(), file.toUri().toString(), "", outputFields, ",", "\"", "UTF-8", false); + + BinDatabase.BinRecord record = db.lookup("5120031234567890"); + assertNotNull(record); + assertEquals("BANKVALUE", record.getValue("bank_name")); + } + + @Test + void testBrandDetectedAsCardType(@TempDir Path tempDir) throws Exception { + Path file = tempDir.resolve("bins.csv"); + try (OutputStream out = Files.newOutputStream(file)) { + out.write(("BIN,BRAND\n" + "512003,MASTERCARD\n").getBytes(StandardCharsets.UTF_8)); + } + + BinDatabase db = new BinDatabase(); + db.load( + new Variables(), file.toUri().toString(), "", new ArrayList<>(), ",", "\"", "UTF-8", true); + + BinDatabase.BinRecord record = db.lookup("5120031234567890"); + assertNotNull(record); + assertEquals("MASTERCARD", record.getCardType()); + } + + @Test + void testUnmatchedCardTypeIsEmpty(@TempDir Path tempDir) throws Exception { + Path file = tempDir.resolve("bins.csv"); + try (OutputStream out = Files.newOutputStream(file)) { + out.write(("BIN,FOO\n" + "512003,MASTERCARD\n").getBytes(StandardCharsets.UTF_8)); + } + + BinDatabase db = new BinDatabase(); + db.load( + new Variables(), file.toUri().toString(), "", new ArrayList<>(), ",", "\"", "UTF-8", true); + + BinDatabase.BinRecord record = db.lookup("5120031234567890"); + assertNotNull(record); + assertEquals("", record.getCardType()); + } + + @Test + void testSkippedRowsCounted(@TempDir Path tempDir) throws Exception { + Path file = tempDir.resolve("bins.csv"); + try (OutputStream out = Files.newOutputStream(file)) { + out.write( + ("BIN,CARDTYPE\n" + + "512003,MASTERCARD\n" + + "123456789012,TOOLONG\n" + + "ABCDEF,NOTNUMERIC\n") + .getBytes(StandardCharsets.UTF_8)); + } + + BinDatabase db = new BinDatabase(); + db.load( + new Variables(), file.toUri().toString(), "", new ArrayList<>(), ",", "\"", "UTF-8", true); + + assertEquals(1, db.size()); + assertEquals(2, db.getSkippedRows()); + } + + @Test + void testNoExtraFieldsUsesEmptyValues(@TempDir Path tempDir) throws Exception { + Path file = tempDir.resolve("bins.csv"); + try (OutputStream out = Files.newOutputStream(file)) { + out.write(("BIN,CARDTYPE\n" + "512003,MASTERCARD\n").getBytes(StandardCharsets.UTF_8)); + } + + BinDatabase db = new BinDatabase(); + db.load( + new Variables(), file.toUri().toString(), "", new ArrayList<>(), ",", "\"", "UTF-8", true); + + BinDatabase.BinRecord record = db.lookup("5120031234567890"); + assertNotNull(record); + assertTrue(record.getExtraValues().isEmpty()); + assertNull(record.getValue("anything")); + } +} diff --git a/plugins/transforms/creditcardvalidator/src/test/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CardSchemeTest.java b/plugins/transforms/creditcardvalidator/src/test/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CardSchemeTest.java new file mode 100644 index 00000000000..c5f3a95535b --- /dev/null +++ b/plugins/transforms/creditcardvalidator/src/test/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CardSchemeTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.creditcardvalidator; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class CardSchemeTest { + + @Test + void testVisaLengths() { + assertTrue(CardScheme.isValidLength("411111", 13)); + assertTrue(CardScheme.isValidLength("411111", 16)); + assertTrue(CardScheme.isValidLength("411111", 19)); + assertFalse(CardScheme.isValidLength("411111", 15)); + assertFalse(CardScheme.isValidLength("411111", 14)); + } + + @Test + void testMastercardLengths() { + assertTrue(CardScheme.isValidLength("511111", 16)); + assertFalse(CardScheme.isValidLength("511111", 15)); + assertTrue(CardScheme.isValidLength("222100", 16)); + } + + @Test + void testAmexLengths() { + assertTrue(CardScheme.isValidLength("341111", 15)); + assertFalse(CardScheme.isValidLength("341111", 16)); + } + + @Test + void testDiscoverLengths() { + assertTrue(CardScheme.isValidLength("601100", 16)); + assertTrue(CardScheme.isValidLength("601100", 19)); + assertTrue(CardScheme.isValidLength("651111", 17)); + assertFalse(CardScheme.isValidLength("601100", 15)); + } + + @Test + void testJcbLengths() { + assertTrue(CardScheme.isValidLength("352811", 16)); + assertTrue(CardScheme.isValidLength("352811", 19)); + assertFalse(CardScheme.isValidLength("352811", 15)); + } + + @Test + void testUnknownSchemeAcceptsAnyLength() { + assertTrue(CardScheme.isValidLength("999999", 16)); + assertTrue(CardScheme.isValidLength("999999", 13)); + } +} diff --git a/plugins/transforms/creditcardvalidator/src/test/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidatorMetaTest.java b/plugins/transforms/creditcardvalidator/src/test/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidatorMetaTest.java index c52538d158a..c39d44bab74 100644 --- a/plugins/transforms/creditcardvalidator/src/test/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidatorMetaTest.java +++ b/plugins/transforms/creditcardvalidator/src/test/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardValidatorMetaTest.java @@ -22,14 +22,24 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.ArrayList; +import java.util.List; +import org.apache.hop.core.HopEnvironment; import org.apache.hop.core.exception.HopTransformException; import org.apache.hop.core.row.IValueMeta; import org.apache.hop.core.row.RowMeta; import org.apache.hop.core.variables.Variables; import org.apache.hop.pipeline.transform.TransformSerializationTestUtil; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; class CreditCardValidatorMetaTest { + + @BeforeAll + static void init() throws Exception { + HopEnvironment.init(); + } + @Test void testLoadSave() throws Exception { CreditCardValidatorMeta meta = @@ -40,6 +50,16 @@ void testLoadSave() throws Exception { assertNotNull(meta.getNotValidMessage()); assertTrue(meta.isOnlyDigits()); assertNotNull(meta.getCardType()); + assertTrue(meta.isUseBinDatabase()); + assertNotNull(meta.getBinFileName()); + assertEquals(",", meta.getBinDelimiter()); + assertEquals("\"", meta.getBinEnclosure()); + assertEquals("UTF-8", meta.getBinEncoding()); + assertTrue(meta.isBinHeaderPresent()); + assertEquals("BIN", meta.getBinCsvColumn()); + assertNotNull(meta.getOutputFields()); + assertEquals(1, meta.getOutputFields().size()); + assertEquals("bank_name", meta.getOutputFields().get(0).getName()); } @Test @@ -55,6 +75,15 @@ void testDefaults() { assertFalse(meta.isOnlyDigits()); assertEquals("card type", meta.getCardType()); assertEquals("not valid message", meta.getNotValidMessage()); + assertFalse(meta.isUseBinDatabase()); + assertEquals("", meta.getBinFileName()); + assertEquals(",", meta.getBinDelimiter()); + assertEquals("\"", meta.getBinEnclosure()); + assertEquals("UTF-8", meta.getBinEncoding()); + assertTrue(meta.isBinHeaderPresent()); + assertEquals("", meta.getBinCsvColumn()); + assertNotNull(meta.getOutputFields()); + assertTrue(meta.getOutputFields().isEmpty()); } @Test @@ -78,4 +107,46 @@ void testGetFields() throws HopTransformException { assertEquals(IValueMeta.TYPE_STRING, rowMeta.getValueMeta(2).getType()); assertEquals("this transform", rowMeta.getValueMeta(2).getOrigin()); } + + @Test + void testGetFieldsExtraOutputType() throws HopTransformException { + CreditCardValidatorMeta meta = new CreditCardValidatorMeta(); + meta.setDefault(); + meta.setUseBinDatabase(true); + meta.setResultFieldName("result"); + meta.setCardType("card type"); + meta.setNotValidMessage("not valid"); + + BinOutputField field = new BinOutputField("my_int", "NUM"); + field.setType(IValueMeta.TYPE_INTEGER); + field.setLength(10); + field.setPrecision(0); + List fields = new ArrayList<>(); + fields.add(field); + meta.setOutputFields(fields); + + RowMeta rowMeta = new RowMeta(); + meta.getFields(rowMeta, "this transform", null, null, new Variables(), null); + assertEquals(4, rowMeta.size()); + assertEquals("my_int", rowMeta.getValueMeta(3).getName()); + assertEquals(IValueMeta.TYPE_INTEGER, rowMeta.getValueMeta(3).getType()); + } + + @Test + void testGetFieldsExtraOutputSkippedWhenBinDatabaseOff() throws HopTransformException { + CreditCardValidatorMeta meta = new CreditCardValidatorMeta(); + meta.setDefault(); + meta.setUseBinDatabase(false); + meta.setResultFieldName("result"); + meta.setCardType("card type"); + meta.setNotValidMessage("not valid"); + + List fields = new ArrayList<>(); + fields.add(new BinOutputField("extra", "NUM")); + meta.setOutputFields(fields); + + RowMeta rowMeta = new RowMeta(); + meta.getFields(rowMeta, "this transform", null, null, new Variables(), null); + assertEquals(3, rowMeta.size()); + } } diff --git a/plugins/transforms/creditcardvalidator/src/test/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardVerifierTest.java b/plugins/transforms/creditcardvalidator/src/test/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardVerifierTest.java index 037bc87227b..2fcb8632670 100644 --- a/plugins/transforms/creditcardvalidator/src/test/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardVerifierTest.java +++ b/plugins/transforms/creditcardvalidator/src/test/java/org/apache/hop/pipeline/transforms/creditcardvalidator/CreditCardVerifierTest.java @@ -19,10 +19,18 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import org.apache.hop.core.variables.Variables; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; class CreditCardVerifierTest { @@ -56,4 +64,29 @@ void testIsNumber() { assertTrue(CreditCardVerifier.isNumber("1")); assertTrue(CreditCardVerifier.isNumber("1.01")); } + + @Test + void testBinDatabaseValidAndTypeSpecificError(@TempDir Path tempDir) throws Exception { + Path file = tempDir.resolve("bins.csv"); + try (OutputStream out = Files.newOutputStream(file)) { + out.write( + ("BIN,CARDTYPE,ISSUERS,COUNTRY\n" + "512003,MASTERCARD,BCA,INDONESIA\n") + .getBytes(StandardCharsets.UTF_8)); + } + + BinDatabase db = new BinDatabase(); + db.load( + new Variables(), file.toUri().toString(), "", new ArrayList<>(), ",", "\"", "UTF-8", true); + + // Valid Luhn number for the BIN 512003 length 16. + ReturnIndicator valid = CreditCardVerifier.checkCC("5120031234567898", db); + assertTrue(valid.CardValid); + assertEquals("MASTERCARD", valid.CardType); + + // Same BIN but failing Luhn -> type-specific message. + ReturnIndicator invalid = CreditCardVerifier.checkCC("5120031234567890", db); + assertFalse(invalid.CardValid); + assertNotNull(invalid.UnValidMsg); + assertTrue(invalid.UnValidMsg.contains("MASTERCARD")); + } } diff --git a/plugins/transforms/creditcardvalidator/src/test/resources/credit-card-validator-transform.xml b/plugins/transforms/creditcardvalidator/src/test/resources/credit-card-validator-transform.xml index e7962b2de48..9f032926303 100644 --- a/plugins/transforms/creditcardvalidator/src/test/resources/credit-card-validator-transform.xml +++ b/plugins/transforms/creditcardvalidator/src/test/resources/credit-card-validator-transform.xml @@ -32,6 +32,19 @@ card type Y not valid message + Y + ${BIN_FILE} + , + " + UTF-8 + Y + BIN + + + bank_name + ISSUERS + + 320