From 41a7174af01d4d7b8b49beaf2df7202cfbe1c5d6 Mon Sep 17 00:00:00 2001 From: mattcasters Date: Wed, 7 Oct 2026 11:37:32 +0200 Subject: [PATCH 1/4] Issue #4193 : Define lifecycle environments in the project configuration Store environment names, descriptions, and placeholder variable defaults in project-config.json, and edit them from the project dialog. --- .../pages/projects/projects-environments.adoc | 25 +- .../environment/EmbeddedEnvironment.java | 79 ++++ .../EmbeddedEnvironmentValidator.java | 148 ++++++++ .../EmbeddedEnvironmentVariable.java | 40 ++ .../project/EmbeddedEnvironmentDialog.java | 343 ++++++++++++++++++ .../apache/hop/projects/project/Project.java | 31 ++ .../hop/projects/project/ProjectDialog.java | 231 ++++++++++++ .../messages/messages_en_US.properties | 34 ++ .../environment/EmbeddedEnvironmentTest.java | 137 +++++++ .../hop/projects/project/ProjectTest.java | 77 ++++ 10 files changed, 1144 insertions(+), 1 deletion(-) create mode 100644 plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironment.java create mode 100644 plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentValidator.java create mode 100644 plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentVariable.java create mode 100644 plugins/misc/projects/src/main/java/org/apache/hop/projects/project/EmbeddedEnvironmentDialog.java create mode 100644 plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentTest.java diff --git a/docs/hop-user-manual/modules/ROOT/pages/projects/projects-environments.adoc b/docs/hop-user-manual/modules/ROOT/pages/projects/projects-environments.adoc index d23aa57c944..47a2564b251 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/projects/projects-environments.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/projects/projects-environments.adoc @@ -36,7 +36,7 @@ image:hop-gui/environment/add-project-menu.png[Add project menu] *From version control* checks out a project from a Git repository instead. -The project properties fields are grouped on four tabs: *Basic*, *Folders*, *Parent project*, and *Variables*. +The project properties fields are grouped on five tabs: *Basic*, *Folders*, *Parent project*, *Variables*, and *Environments*. image:hop-gui/environment/create-project-dialog.png[Project Properties Basic tab,width="90%"] @@ -107,6 +107,29 @@ image:hop-gui/environment/create-project-dialog-variables-tab.png[Project Proper |Project variables to set|A list of variable names, values and variable descriptions to use with this project|No|No| |=== +== Embedded environments + +The *Environments* tab stores zero, one, or more lifecycle environment definitions in `project-config.json`. +These definitions are meant to be checked in with the project. +Each one has a name, a description, and three lists of variables. +A variable name belongs to only one of the three lists. +The definition does not contain configuration files. + +The *Default value* of a variable is a placeholder, for example `change-to-your-password` or `specify the folder on the system where the files are`. +Do not put a real password, host, or path in the project file. + +.Embedded environment variable lists +[cols="20%,80%",options="header"] +|=== +|List|What it is for +|Variables|Optional variables. The default is documentation for people who use the project. +|Mandatory|Variables a person has to fill in on their own computer. The default is still only a placeholder. +|Secrets|Secret variables such as passwords and tokens. The default is a placeholder, not the secret. +|=== + +*Add*, *Edit...*, and *Delete* change the list. +*Edit...* sets the name, the description, and the three variable lists. + After creating a project the user interface will switch to it and ask if you want to create an environment. == Create an environment diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironment.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironment.java new file mode 100644 index 00000000000..b05010d82e5 --- /dev/null +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironment.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.projects.environment; + +import java.util.ArrayList; +import java.util.List; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +/** + * Lifecycle environment definition stored in {@code project-config.json}. + * + *

Holds the name, description, and placeholder variable defaults that are shared through version + * control. Configuration files and real values stay on the computer that runs Hop. + */ +@Getter +@Setter +@NoArgsConstructor +public class EmbeddedEnvironment { + + private String name; + + private String description; + + /** Optional variables. Applied as defaults. Not written to a local configuration file. */ + private List variables = new ArrayList<>(); + + /** Values a person must fill in. Written to the local configuration file when one is created. */ + private List mandatoryVariables = new ArrayList<>(); + + /** Secrets. Written to the local configuration file when one is created. Placeholders only. */ + private List secretVariables = new ArrayList<>(); + + /** + * @return a deep copy + */ + public EmbeddedEnvironment copy() { + EmbeddedEnvironment copy = new EmbeddedEnvironment(); + copy.name = name; + copy.description = description; + copy.variables = copyVariables(variables); + copy.mandatoryVariables = copyVariables(mandatoryVariables); + copy.secretVariables = copyVariables(secretVariables); + return copy; + } + + private static List copyVariables( + List source) { + List copy = new ArrayList<>(); + if (source == null) { + return copy; + } + for (EmbeddedEnvironmentVariable variable : source) { + if (variable == null) { + continue; + } + copy.add( + new EmbeddedEnvironmentVariable( + variable.getName(), variable.getDefaultValue(), variable.getDescription())); + } + return copy; + } +} diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentValidator.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentValidator.java new file mode 100644 index 00000000000..4a252bcb751 --- /dev/null +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentValidator.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.projects.environment; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.apache.commons.lang3.StringUtils; + +/** Checks and tidies embedded environment definitions before they are written. */ +public final class EmbeddedEnvironmentValidator { + + private EmbeddedEnvironmentValidator() {} + + /** + * Trim names and drop variables that have no name. A blank description or default is stored as + * null so it is left out of {@code project-config.json}. + * + * @param environment definition to tidy, ignored when null + */ + public static void normalize(EmbeddedEnvironment environment) { + if (environment == null) { + return; + } + environment.setName(StringUtils.trimToNull(environment.getName())); + environment.setDescription(StringUtils.trimToNull(environment.getDescription())); + environment.setVariables(normalizeVariables(environment.getVariables())); + environment.setMandatoryVariables(normalizeVariables(environment.getMandatoryVariables())); + environment.setSecretVariables(normalizeVariables(environment.getSecretVariables())); + } + + /** + * @param environment definition to check + * @return true when the name is missing + */ + public static boolean missingName(EmbeddedEnvironment environment) { + return environment == null || StringUtils.isBlank(environment.getName()); + } + + /** + * @param environment definition whose three lists are checked together + * @return the first variable name that appears more than once, or null + */ + public static String duplicateVariableName(EmbeddedEnvironment environment) { + if (environment == null) { + return null; + } + Set seen = new HashSet<>(); + for (EmbeddedEnvironmentVariable variable : allVariables(environment)) { + if (variable == null || StringUtils.isBlank(variable.getName())) { + continue; + } + if (!seen.add(variable.getName().trim())) { + return variable.getName().trim(); + } + } + return null; + } + + /** + * @param environments definitions in one project + * @return the first environment name that appears more than once, or null. Blank names are + * ignored. + */ + public static String duplicateEnvironmentName(List environments) { + if (environments == null) { + return null; + } + Set seen = new HashSet<>(); + for (EmbeddedEnvironment environment : environments) { + if (environment == null || StringUtils.isBlank(environment.getName())) { + continue; + } + String name = environment.getName().trim(); + if (!seen.add(name)) { + return name; + } + } + return null; + } + + /** + * @param name candidate environment name + * @param otherNames names already used by the other definitions in the project + * @return true when {@code name} matches one of {@code otherNames} + */ + public static boolean nameTaken(String name, List otherNames) { + if (StringUtils.isBlank(name) || otherNames == null) { + return false; + } + String trimmed = name.trim(); + for (String other : otherNames) { + if (trimmed.equals(StringUtils.trimToEmpty(other))) { + return true; + } + } + return false; + } + + private static List allVariables(EmbeddedEnvironment environment) { + List all = new ArrayList<>(); + addAll(all, environment.getVariables()); + addAll(all, environment.getMandatoryVariables()); + addAll(all, environment.getSecretVariables()); + return all; + } + + private static void addAll( + List target, List source) { + if (source != null) { + target.addAll(source); + } + } + + private static List normalizeVariables( + List source) { + List normalized = new ArrayList<>(); + if (source == null) { + return normalized; + } + for (EmbeddedEnvironmentVariable variable : source) { + if (variable == null || StringUtils.isBlank(variable.getName())) { + continue; + } + variable.setName(variable.getName().trim()); + variable.setDefaultValue(StringUtils.trimToNull(variable.getDefaultValue())); + variable.setDescription(StringUtils.trimToNull(variable.getDescription())); + normalized.add(variable); + } + return normalized; + } +} diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentVariable.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentVariable.java new file mode 100644 index 00000000000..9f078724667 --- /dev/null +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentVariable.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.projects.environment; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +/** + * A variable declared on an embedded environment. {@code defaultValue} is a placeholder checked in + * with the project, not a live value. + */ +@Getter +@Setter +@NoArgsConstructor +@AllArgsConstructor +public class EmbeddedEnvironmentVariable { + + private String name; + + private String defaultValue; + + private String description; +} diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/EmbeddedEnvironmentDialog.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/EmbeddedEnvironmentDialog.java new file mode 100644 index 00000000000..0e9cd054dc9 --- /dev/null +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/EmbeddedEnvironmentDialog.java @@ -0,0 +1,343 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.projects.project; + +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.Const; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.projects.environment.EmbeddedEnvironment; +import org.apache.hop.projects.environment.EmbeddedEnvironmentValidator; +import org.apache.hop.projects.environment.EmbeddedEnvironmentVariable; +import org.apache.hop.projects.util.Defaults; +import org.apache.hop.ui.core.ConstUi; +import org.apache.hop.ui.core.PropsUi; +import org.apache.hop.ui.core.dialog.BaseDialog; +import org.apache.hop.ui.core.dialog.MessageBox; +import org.apache.hop.ui.core.gui.GuiResource; +import org.apache.hop.ui.core.gui.WindowProperty; +import org.apache.hop.ui.core.widget.ColumnInfo; +import org.apache.hop.ui.core.widget.NamingSchemeTypes; +import org.apache.hop.ui.core.widget.TableView; +import org.apache.hop.ui.core.widget.TextVar; +import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; +import org.apache.hop.ui.util.HelpUtils; +import org.eclipse.swt.SWT; +import org.eclipse.swt.custom.CTabFolder; +import org.eclipse.swt.custom.CTabItem; +import org.eclipse.swt.layout.FormAttachment; +import org.eclipse.swt.layout.FormData; +import org.eclipse.swt.layout.FormLayout; +import org.eclipse.swt.widgets.Button; +import org.eclipse.swt.widgets.Composite; +import org.eclipse.swt.widgets.Control; +import org.eclipse.swt.widgets.Dialog; +import org.eclipse.swt.widgets.Label; +import org.eclipse.swt.widgets.Shell; +import org.eclipse.swt.widgets.TableItem; +import org.eclipse.swt.widgets.Text; + +/** Edits one embedded environment: name, description, and the three variable lists. */ +public class EmbeddedEnvironmentDialog extends Dialog { + private static final Class PKG = EmbeddedEnvironmentDialog.class; + + private final EmbeddedEnvironment environment; + private final List otherNames; + private final IVariables variables; + private final PropsUi props; + + private Shell shell; + private TextVar wName; + private Text wDescription; + private TableView wVariables; + private TableView wMandatoryVariables; + private TableView wSecretVariables; + + private EmbeddedEnvironment returnValue; + + public EmbeddedEnvironmentDialog( + Shell parent, + EmbeddedEnvironment environment, + List otherNames, + IVariables variables) { + super(parent, SWT.DIALOG_TRIM | SWT.APPLICATION_MODAL | SWT.RESIZE); + this.environment = environment; + this.otherNames = otherNames == null ? List.of() : otherNames; + this.variables = variables; + this.props = PropsUi.getInstance(); + } + + /** + * @return the edited environment, or null when cancelled. The environment passed to the + * constructor is updated only when the user confirms. + */ + public EmbeddedEnvironment open() { + Shell parent = getParent(); + shell = new Shell(parent, SWT.DIALOG_TRIM | SWT.APPLICATION_MODAL | SWT.RESIZE); + shell.setImage( + GuiResource.getInstance() + .getImage( + "environment.svg", + PKG.getClassLoader(), + ConstUi.SMALL_ICON_SIZE, + ConstUi.SMALL_ICON_SIZE)); + PropsUi.setLook(shell); + + int margin = PropsUi.getMargin() + 2; + FormLayout formLayout = new FormLayout(); + formLayout.marginWidth = PropsUi.getFormMargin(); + formLayout.marginHeight = PropsUi.getFormMargin(); + shell.setLayout(formLayout); + shell.setText(BaseMessages.getString(PKG, "EmbeddedEnvironmentDialog.Shell.Name")); + + Button wOk = new Button(shell, SWT.PUSH); + wOk.setText(BaseMessages.getString(PKG, "System.Button.OK")); + wOk.addListener(SWT.Selection, event -> ok()); + Button wCancel = new Button(shell, SWT.PUSH); + wCancel.setText(BaseMessages.getString(PKG, "System.Button.Cancel")); + wCancel.addListener(SWT.Selection, event -> cancel()); + BaseTransformDialog.positionBottomButtons(shell, new Button[] {wOk, wCancel}, margin * 3, null); + HelpUtils.createHelpButton(shell, Const.getDocUrl(Defaults.DOCUMENTATION_URI)); + + int middle = props.getMiddlePct(); + Control last = addName(margin, middle); + last = addDescription(margin, middle, last); + + CTabFolder tabs = new CTabFolder(shell, SWT.BORDER); + PropsUi.setLook(tabs); + FormData fdTabs = new FormData(); + fdTabs.left = new FormAttachment(0, 0); + fdTabs.top = new FormAttachment(last, margin); + fdTabs.right = new FormAttachment(100, 0); + fdTabs.bottom = new FormAttachment(wOk, -margin * 2); + tabs.setLayoutData(fdTabs); + + wVariables = + addVariableTab( + tabs, + "EmbeddedEnvironmentDialog.Tab.Variables", + environment.getVariables(), + "EmbeddedEnvironmentDialog.Tab.Variables.Tooltip"); + wMandatoryVariables = + addVariableTab( + tabs, + "EmbeddedEnvironmentDialog.Tab.Mandatory", + environment.getMandatoryVariables(), + "EmbeddedEnvironmentDialog.Tab.Mandatory.Tooltip"); + wSecretVariables = + addVariableTab( + tabs, + "EmbeddedEnvironmentDialog.Tab.Secrets", + environment.getSecretVariables(), + "EmbeddedEnvironmentDialog.Tab.Secrets.Tooltip"); + tabs.setSelection(0); + + wName.setText(Const.NVL(environment.getName(), "")); + wDescription.setText(Const.NVL(environment.getDescription(), "")); + + shell.setMinimumSize(700, 500); + shell.setDefaultButton(wOk); + wName.setFocus(); + BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel()); + return returnValue; + } + + private Control addName(int margin, int middle) { + Label label = new Label(shell, SWT.RIGHT); + PropsUi.setLook(label); + label.setText(BaseMessages.getString(PKG, "EmbeddedEnvironmentDialog.Label.Name")); + FormData fdLabel = new FormData(); + fdLabel.left = new FormAttachment(0, 0); + fdLabel.right = new FormAttachment(middle, 0); + fdLabel.top = new FormAttachment(0, margin); + label.setLayoutData(fdLabel); + + wName = + new TextVar(variables, shell, SWT.SINGLE | SWT.BORDER | SWT.LEFT) + .asNameField(NamingSchemeTypes.HOP_METADATA); + PropsUi.setLook(wName); + FormData fdName = new FormData(); + fdName.left = new FormAttachment(middle, margin); + fdName.right = new FormAttachment(100, 0); + fdName.top = new FormAttachment(label, 0, SWT.CENTER); + wName.setLayoutData(fdName); + return wName; + } + + private Control addDescription(int margin, int middle, Control previous) { + Label label = new Label(shell, SWT.RIGHT); + PropsUi.setLook(label); + label.setText(BaseMessages.getString(PKG, "EmbeddedEnvironmentDialog.Label.Description")); + FormData fdLabel = new FormData(); + fdLabel.left = new FormAttachment(0, 0); + fdLabel.right = new FormAttachment(middle, 0); + fdLabel.top = new FormAttachment(previous, margin); + label.setLayoutData(fdLabel); + + wDescription = new Text(shell, SWT.SINGLE | SWT.BORDER | SWT.LEFT); + PropsUi.setLook(wDescription); + FormData fdDescription = new FormData(); + fdDescription.left = new FormAttachment(middle, margin); + fdDescription.right = new FormAttachment(100, 0); + fdDescription.top = new FormAttachment(label, 0, SWT.CENTER); + wDescription.setLayoutData(fdDescription); + return wDescription; + } + + private TableView addVariableTab( + CTabFolder folder, String tabKey, List rows, String tooltipKey) { + CTabItem tab = new CTabItem(folder, SWT.NONE); + tab.setText(BaseMessages.getString(PKG, tabKey)); + tab.setToolTipText(BaseMessages.getString(PKG, tooltipKey)); + Composite comp = new Composite(folder, SWT.NONE); + PropsUi.setLook(comp); + FormLayout layout = new FormLayout(); + layout.marginWidth = PropsUi.getFormMargin(); + layout.marginHeight = PropsUi.getFormMargin(); + comp.setLayout(layout); + tab.setControl(comp); + + int size = rows == null ? 0 : rows.size(); + ColumnInfo[] columnInfo = variableColumns(); + TableView table = + new TableView( + variables, comp, SWT.BORDER, columnInfo, Math.max(size, 3), event -> {}, props); + PropsUi.setLook(table); + FormData fdTable = new FormData(); + fdTable.left = new FormAttachment(0, 0); + fdTable.right = new FormAttachment(100, 0); + fdTable.top = new FormAttachment(0, 0); + fdTable.bottom = new FormAttachment(100, 0); + table.setLayoutData(fdTable); + fillVariables(table, rows); + return table; + } + + private ColumnInfo[] variableColumns() { + ColumnInfo[] columnInfo = + new ColumnInfo[] { + new ColumnInfo( + BaseMessages.getString(PKG, "EmbeddedEnvironmentDialog.Column.Name"), + ColumnInfo.COLUMN_TYPE_TEXT, + false, + false), + new ColumnInfo( + BaseMessages.getString(PKG, "EmbeddedEnvironmentDialog.Column.Default"), + ColumnInfo.COLUMN_TYPE_TEXT, + false, + false), + new ColumnInfo( + BaseMessages.getString(PKG, "EmbeddedEnvironmentDialog.Column.Description"), + ColumnInfo.COLUMN_TYPE_TEXT, + false, + false), + }; + columnInfo[0].setUsingVariables(true); + columnInfo[0].setNamingSchemeType(NamingSchemeTypes.HOP_VARIABLE); + columnInfo[1].setUsingVariables(true); + columnInfo[1].setToolTip( + BaseMessages.getString(PKG, "EmbeddedEnvironmentDialog.Column.Default.Tooltip")); + return columnInfo; + } + + private void fillVariables(TableView table, List rows) { + if (rows == null) { + return; + } + for (int i = 0; i < rows.size(); i++) { + EmbeddedEnvironmentVariable variable = rows.get(i); + TableItem item = table.table.getItem(i); + item.setText(1, Const.NVL(variable.getName(), "")); + item.setText(2, Const.NVL(variable.getDefaultValue(), "")); + item.setText(3, Const.NVL(variable.getDescription(), "")); + } + table.setRowNums(); + table.optWidth(true); + } + + private List readVariables(TableView table) { + List rows = new ArrayList<>(); + for (int i = 0; i < table.nrNonEmpty(); i++) { + TableItem item = table.getNonEmpty(i); + if (StringUtils.isBlank(item.getText(1))) { + continue; + } + rows.add(new EmbeddedEnvironmentVariable(item.getText(1), item.getText(2), item.getText(3))); + } + return rows; + } + + private void ok() { + EmbeddedEnvironment edited = new EmbeddedEnvironment(); + edited.setName(wName.getText()); + edited.setDescription(wDescription.getText()); + edited.setVariables(readVariables(wVariables)); + edited.setMandatoryVariables(readVariables(wMandatoryVariables)); + edited.setSecretVariables(readVariables(wSecretVariables)); + EmbeddedEnvironmentValidator.normalize(edited); + + if (EmbeddedEnvironmentValidator.missingName(edited)) { + showError( + "EmbeddedEnvironmentDialog.MissingName.Header", + "EmbeddedEnvironmentDialog.MissingName.Message"); + return; + } + String duplicateVariable = EmbeddedEnvironmentValidator.duplicateVariableName(edited); + if (duplicateVariable != null) { + showError( + "EmbeddedEnvironmentDialog.DuplicateVariable.Header", + "EmbeddedEnvironmentDialog.DuplicateVariable.Message", + duplicateVariable); + return; + } + if (EmbeddedEnvironmentValidator.nameTaken(edited.getName(), otherNames)) { + showError( + "EmbeddedEnvironmentDialog.DuplicateName.Header", + "EmbeddedEnvironmentDialog.DuplicateName.Message", + edited.getName()); + return; + } + + environment.setName(edited.getName()); + environment.setDescription(edited.getDescription()); + environment.setVariables(edited.getVariables()); + environment.setMandatoryVariables(edited.getMandatoryVariables()); + environment.setSecretVariables(edited.getSecretVariables()); + returnValue = environment; + dispose(); + } + + private void showError(String headerKey, String messageKey, String... args) { + MessageBox box = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR); + box.setText(BaseMessages.getString(PKG, headerKey)); + box.setMessage(BaseMessages.getString(PKG, messageKey, (Object[]) args)); + box.open(); + } + + private void cancel() { + returnValue = null; + dispose(); + } + + private void dispose() { + props.setScreen(new WindowProperty(shell)); + shell.dispose(); + } +} diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/Project.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/Project.java index f06bb1b9485..0e4a67f0279 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/Project.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/Project.java @@ -64,6 +64,7 @@ import org.apache.hop.pipeline.transform.TransformMeta; import org.apache.hop.projects.config.ProjectsConfig; import org.apache.hop.projects.config.ProjectsConfigSingleton; +import org.apache.hop.projects.environment.EmbeddedEnvironment; import org.apache.hop.projects.util.Defaults; import org.apache.hop.projects.util.ProjectsUtil; import org.apache.hop.workflow.WorkflowMeta; @@ -113,6 +114,13 @@ public class Project extends ConfigFile implements IConfigFile { */ private List parentProjectFolders; + /** + * Lifecycle environment definitions checked in with the project. Names, descriptions, and + * placeholder variable defaults only. Real values and configuration files stay on the computer + * that runs Hop. + */ + private List embeddedEnvironments; + @JsonIgnore private MultiMetadataProvider metadataProvider; @JsonIgnore private List pipelinePaths; @JsonIgnore private List workflowPaths; @@ -128,6 +136,7 @@ public Project() { autoExportMetadata = false; autoExportMetadataFilename = ""; parentProjectFolders = new ArrayList<>(); + embeddedEnvironments = new ArrayList<>(); } public Project(String configFilename) { @@ -192,12 +201,34 @@ public void readFromFile() throws HopException { project.parentProjectFolders != null ? new ArrayList<>(project.parentProjectFolders) : new ArrayList<>(); + this.embeddedEnvironments = + project.embeddedEnvironments != null + ? new ArrayList<>(project.embeddedEnvironments) + : new ArrayList<>(); } catch (Exception e) { throw new HopException( "Error saving project configuration to file '" + configFilename + "'", e); } } + /** + * Find an embedded environment by its exact name. + * + * @param name environment name + * @return the definition, or null when there is none with that name + */ + public EmbeddedEnvironment findEmbeddedEnvironment(String name) { + if (name == null || embeddedEnvironments == null) { + return null; + } + for (EmbeddedEnvironment environment : embeddedEnvironments) { + if (environment != null && name.equals(environment.getName())) { + return environment; + } + } + return null; + } + public void modifyVariables( IVariables variables, ProjectConfig projectConfig, diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ProjectDialog.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ProjectDialog.java index 2f012aa5ec6..aafdb33892c 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ProjectDialog.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ProjectDialog.java @@ -18,6 +18,7 @@ package org.apache.hop.projects.project; import java.io.File; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import lombok.Getter; @@ -35,6 +36,8 @@ import org.apache.hop.i18n.BaseMessages; import org.apache.hop.projects.config.ProjectsConfig; import org.apache.hop.projects.config.ProjectsConfigSingleton; +import org.apache.hop.projects.environment.EmbeddedEnvironment; +import org.apache.hop.projects.environment.EmbeddedEnvironmentValidator; import org.apache.hop.projects.gui.ProjectsGuiPlugin; import org.apache.hop.projects.util.Defaults; import org.apache.hop.projects.util.ProjectRenameBlockedException; @@ -104,6 +107,13 @@ public class ProjectDialog extends Dialog { private Button wEnforceHomeExecution; private TableView wVariables; private TableView wParentFolders; + private TableView wEnvironments; + private Button wAddEnvironment; + private Button wEditEnvironment; + private Button wDeleteEnvironment; + + /** Working copy of the embedded environments. Applied to the project on OK. */ + private final List embeddedEnvironments = new ArrayList<>(); private final IVariables variables; @@ -194,6 +204,7 @@ public String open() { createFoldersTab(wTabFolder, margin); createParentProjectTab(wTabFolder, margin); createVariablesTab(wTabFolder, margin); + createEnvironmentsTab(wTabFolder, margin); wParentProject.addModifyListener( e -> { @@ -686,6 +697,154 @@ private void createVariablesTab(CTabFolder folder, int margin) { wVariables.setLayoutData(fdVariables); } + private void createEnvironmentsTab(CTabFolder folder, int margin) { + Composite comp = createTab(folder, "ProjectDialog.Tab.Environments"); + + Label explanation = new Label(comp, SWT.LEFT | SWT.WRAP); + PropsUi.setLook(explanation); + explanation.setText(BaseMessages.getString(PKG, "ProjectDialog.Environments.Explanation")); + FormData fdExplanation = new FormData(); + fdExplanation.left = new FormAttachment(0, 0); + fdExplanation.right = new FormAttachment(100, 0); + fdExplanation.top = new FormAttachment(0, 0); + explanation.setLayoutData(fdExplanation); + + wAddEnvironment = new Button(comp, SWT.PUSH); + wAddEnvironment.setText(BaseMessages.getString(PKG, "ProjectDialog.Button.AddEnvironment")); + wAddEnvironment.addListener(SWT.Selection, event -> addEmbeddedEnvironment()); + wEditEnvironment = new Button(comp, SWT.PUSH); + wEditEnvironment.setText(BaseMessages.getString(PKG, "ProjectDialog.Button.EditEnvironment")); + wEditEnvironment.addListener(SWT.Selection, event -> editEmbeddedEnvironment()); + wDeleteEnvironment = new Button(comp, SWT.PUSH); + wDeleteEnvironment.setText( + BaseMessages.getString(PKG, "ProjectDialog.Button.DeleteEnvironment")); + wDeleteEnvironment.addListener(SWT.Selection, event -> deleteEmbeddedEnvironment()); + BaseTransformDialog.positionBottomButtons( + comp, new Button[] {wAddEnvironment, wEditEnvironment, wDeleteEnvironment}, margin, null); + + ColumnInfo[] columnInfo = + new ColumnInfo[] { + new ColumnInfo( + BaseMessages.getString(PKG, "ProjectDialog.Environments.Column.Name"), + ColumnInfo.COLUMN_TYPE_TEXT, + false, + true), + new ColumnInfo( + BaseMessages.getString(PKG, "ProjectDialog.Environments.Column.Description"), + ColumnInfo.COLUMN_TYPE_TEXT, + false, + true), + }; + wEnvironments = + new TableView( + variables, + comp, + SWT.BORDER | SWT.FULL_SELECTION | SWT.SINGLE, + columnInfo, + 1, + true, + null, + props, + true, + null, + false, + false); + PropsUi.setLook(wEnvironments); + FormData fdEnvironments = new FormData(); + fdEnvironments.left = new FormAttachment(0, 0); + fdEnvironments.right = new FormAttachment(100, 0); + fdEnvironments.top = new FormAttachment(explanation, margin); + fdEnvironments.bottom = new FormAttachment(wAddEnvironment, -margin); + wEnvironments.setLayoutData(fdEnvironments); + wEnvironments.table.addListener(SWT.DefaultSelection, event -> editEmbeddedEnvironment()); + } + + private void addEmbeddedEnvironment() { + if (!environmentsEditable()) { + return; + } + EmbeddedEnvironment created = new EmbeddedEnvironment(); + EmbeddedEnvironmentDialog dialog = + new EmbeddedEnvironmentDialog(shell, created, environmentNamesExcept(-1), variables); + if (dialog.open() != null) { + embeddedEnvironments.add(created); + refreshEmbeddedEnvironments(embeddedEnvironments.size() - 1); + needingProjectRefresh = true; + } + } + + private void editEmbeddedEnvironment() { + if (!environmentsEditable()) { + return; + } + int index = wEnvironments.getSelectionIndex(); + if (index < 0 || index >= embeddedEnvironments.size()) { + return; + } + EmbeddedEnvironment editing = embeddedEnvironments.get(index).copy(); + EmbeddedEnvironmentDialog dialog = + new EmbeddedEnvironmentDialog(shell, editing, environmentNamesExcept(index), variables); + if (dialog.open() != null) { + embeddedEnvironments.set(index, editing); + refreshEmbeddedEnvironments(index); + needingProjectRefresh = true; + } + } + + private void deleteEmbeddedEnvironment() { + if (!environmentsEditable()) { + return; + } + int index = wEnvironments.getSelectionIndex(); + if (index < 0 || index >= embeddedEnvironments.size()) { + return; + } + EmbeddedEnvironment selected = embeddedEnvironments.get(index); + MessageBox box = new MessageBox(shell, SWT.YES | SWT.NO | SWT.ICON_QUESTION); + box.setText(BaseMessages.getString(PKG, "ProjectDialog.Environments.Delete.Header")); + box.setMessage( + BaseMessages.getString( + PKG, "ProjectDialog.Environments.Delete.Message", Const.NVL(selected.getName(), ""))); + if ((box.open() & SWT.YES) == 0) { + return; + } + embeddedEnvironments.remove(index); + refreshEmbeddedEnvironments(Math.min(index, embeddedEnvironments.size() - 1)); + needingProjectRefresh = true; + } + + private boolean environmentsEditable() { + return wAddEnvironment != null && wAddEnvironment.isEnabled(); + } + + /** Names of every embedded environment except the row being edited. {@code -1} excludes none. */ + private List environmentNamesExcept(int exceptIndex) { + List names = new ArrayList<>(); + for (int i = 0; i < embeddedEnvironments.size(); i++) { + if (i == exceptIndex) { + continue; + } + String name = embeddedEnvironments.get(i).getName(); + if (StringUtils.isNotEmpty(name)) { + names.add(name); + } + } + return names; + } + + private void refreshEmbeddedEnvironments(int selectIndex) { + wEnvironments.table.removeAll(); + for (EmbeddedEnvironment environment : embeddedEnvironments) { + wEnvironments.add( + Const.NVL(environment.getName(), ""), Const.NVL(environment.getDescription(), "")); + } + wEnvironments.setRowNums(); + wEnvironments.optWidth(true); + if (selectIndex >= 0 && selectIndex < wEnvironments.table.getItemCount()) { + wEnvironments.table.select(selectIndex); + } + } + /** * Automatically select read-only when the home folder is a VFS archive URI (zip/jar/tar/...). The * user can still uncheck the option, or check it manually for other cases (http://, read-only @@ -722,6 +881,14 @@ private void updateReadOnlyWidgets() { wEnforceHomeExecution.setEnabled(editable); wVariables.setEnabled(editable); wVariables.setReadonly(!editable); + if (wEnvironments != null) { + wEnvironments.setEnabled(editable); + } + if (wAddEnvironment != null) { + wAddEnvironment.setEnabled(editable); + wEditEnvironment.setEnabled(editable); + wDeleteEnvironment.setEnabled(editable); + } updateAutoExportMetadataWidgets(); updateParentFolderWidgets(); } @@ -1005,6 +1172,10 @@ private void ok() { } } + if (!validateEmbeddedEnvironments()) { + return; + } + getInfo(project, projectConfig); returnValue = projectConfig.getProjectName(); dispose(); @@ -1103,6 +1274,16 @@ private void getData() { } wParentFolders.setRowNums(); wParentFolders.optWidth(true); + + embeddedEnvironments.clear(); + if (project.getEmbeddedEnvironments() != null) { + for (EmbeddedEnvironment environment : project.getEmbeddedEnvironments()) { + if (environment != null) { + embeddedEnvironments.add(environment.copy()); + } + } + } + refreshEmbeddedEnvironments(-1); } private void getInfo(Project project, ProjectConfig projectConfig) throws HopException { @@ -1152,6 +1333,12 @@ private void getInfo(Project project, ProjectConfig projectConfig) throws HopExc project.getParentProjectFolders().add(parentFolder); } + List environments = new ArrayList<>(); + for (EmbeddedEnvironment environment : embeddedEnvironments) { + environments.add(environment.copy()); + } + project.setEmbeddedEnvironments(environments); + if (StringUtils.isNotEmpty(projectConfig.getProjectHome()) && StringUtils.isNotEmpty(projectConfig.configFilename)) { try { @@ -1172,6 +1359,50 @@ private void getInfo(Project project, ProjectConfig projectConfig) throws HopExc } } + /** + * @return false when the definitions cannot be saved. Read-only projects skip the check because + * the project file is not written. + */ + private boolean validateEmbeddedEnvironments() { + if (wReadOnly.getSelection()) { + return true; + } + for (EmbeddedEnvironment environment : embeddedEnvironments) { + EmbeddedEnvironmentValidator.normalize(environment); + if (EmbeddedEnvironmentValidator.missingName(environment)) { + showEnvironmentError( + "ProjectDialog.Environments.MissingName.Header", + "ProjectDialog.Environments.MissingName.Message"); + return false; + } + String duplicateVariable = EmbeddedEnvironmentValidator.duplicateVariableName(environment); + if (duplicateVariable != null) { + showEnvironmentError( + "ProjectDialog.Environments.DuplicateVariable.Header", + "ProjectDialog.Environments.DuplicateVariable.Message", + duplicateVariable); + return false; + } + } + String duplicateName = + EmbeddedEnvironmentValidator.duplicateEnvironmentName(embeddedEnvironments); + if (duplicateName != null) { + showEnvironmentError( + "ProjectDialog.Environments.DuplicateName.Header", + "ProjectDialog.Environments.DuplicateName.Message", + duplicateName); + return false; + } + return true; + } + + private void showEnvironmentError(String headerKey, String messageKey, String... args) { + MessageBox box = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR); + box.setText(BaseMessages.getString(PKG, headerKey)); + box.setMessage(BaseMessages.getString(PKG, messageKey, (Object[]) args)); + box.open(); + } + private static boolean isYes(String value) { return "Y".equalsIgnoreCase(Const.NVL(value, "")); } diff --git a/plugins/misc/projects/src/main/resources/org/apache/hop/projects/project/messages/messages_en_US.properties b/plugins/misc/projects/src/main/resources/org/apache/hop/projects/project/messages/messages_en_US.properties index ea42215d19b..34fa382e5cf 100644 --- a/plugins/misc/projects/src/main/resources/org/apache/hop/projects/project/messages/messages_en_US.properties +++ b/plugins/misc/projects/src/main/resources/org/apache/hop/projects/project/messages/messages_en_US.properties @@ -87,3 +87,37 @@ ProjectRename.Blocked.ReadOnly=- ''{0}'' uses ''{1}'' as its parent project, but ProjectRename.Blocked.Unreadable=- ''{0}'': its configuration can''t be read, so it can''t be checked whether it uses ''{1}'' as its parent project. Reason: {2} ProjectRename.SaveFailed.Message=Project ''{0}'' wasn''t renamed to ''{1}'': the parent project of ''{2}'' couldn''t be saved. The rename was undone. ProjectRename.SaveFailed.NotRestored=Project ''{0}'' wasn''t renamed to ''{1}'': the parent project of ''{2}'' couldn''t be saved. The rename was undone, but these projects couldn''t be restored and still use ''{1}'' as their parent project: {3} +ProjectDialog.Tab.Environments=Environments +ProjectDialog.Environments.Explanation=These definitions are stored in the project configuration and can be committed. Enter placeholder default values, not real passwords, hosts, or paths. Configuration files are not stored here. +ProjectDialog.Environments.Column.Name=Name +ProjectDialog.Environments.Column.Description=Description +ProjectDialog.Button.AddEnvironment=Add +ProjectDialog.Button.EditEnvironment=Edit... +ProjectDialog.Button.DeleteEnvironment=Delete +ProjectDialog.Environments.Delete.Header=Delete environment? +ProjectDialog.Environments.Delete.Message=Delete the environment definition ''{0}'' from this project? +ProjectDialog.Environments.MissingName.Header=Environment name +ProjectDialog.Environments.MissingName.Message=Give the environment a name. +ProjectDialog.Environments.DuplicateName.Header=Environment name +ProjectDialog.Environments.DuplicateName.Message=An environment named ''{0}'' is already defined in this project. +ProjectDialog.Environments.DuplicateVariable.Header=Variable name +ProjectDialog.Environments.DuplicateVariable.Message=Variable ''{0}'' is listed more than once. A name can be in only one of the three lists. +EmbeddedEnvironmentDialog.Shell.Name=Environment definition +EmbeddedEnvironmentDialog.Label.Name=Name +EmbeddedEnvironmentDialog.Label.Description=Description +EmbeddedEnvironmentDialog.Tab.Variables=Variables +EmbeddedEnvironmentDialog.Tab.Variables.Tooltip=Optional variables. The default is a placeholder and is not a value for one computer. +EmbeddedEnvironmentDialog.Tab.Mandatory=Mandatory +EmbeddedEnvironmentDialog.Tab.Mandatory.Tooltip=Variables a person has to fill in on their own computer. The default is a placeholder. +EmbeddedEnvironmentDialog.Tab.Secrets=Secrets +EmbeddedEnvironmentDialog.Tab.Secrets.Tooltip=Secret variables. Store a placeholder such as change-to-your-password, not the real secret. +EmbeddedEnvironmentDialog.Column.Name=Name +EmbeddedEnvironmentDialog.Column.Default=Default value +EmbeddedEnvironmentDialog.Column.Default.Tooltip=Placeholder checked in with the project, for example change-to-your-password or specify the folder on the system where the files are. Not a real value. +EmbeddedEnvironmentDialog.Column.Description=Description +EmbeddedEnvironmentDialog.MissingName.Header=Environment name +EmbeddedEnvironmentDialog.MissingName.Message=Give the environment a name. +EmbeddedEnvironmentDialog.DuplicateVariable.Header=Variable name +EmbeddedEnvironmentDialog.DuplicateVariable.Message=Variable ''{0}'' is listed more than once. A name can be in only one of the three lists. +EmbeddedEnvironmentDialog.DuplicateName.Header=Environment name +EmbeddedEnvironmentDialog.DuplicateName.Message=An environment named ''{0}'' is already defined in this project. diff --git a/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentTest.java b/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentTest.java new file mode 100644 index 00000000000..00f2eeb7356 --- /dev/null +++ b/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentTest.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.projects.environment; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.Test; + +class EmbeddedEnvironmentTest { + + @Test + void normalizeDropsBlankNamesAndTrims() { + EmbeddedEnvironment environment = new EmbeddedEnvironment(); + environment.setName(" dev "); + environment.setDescription(" "); + environment + .getVariables() + .add(new EmbeddedEnvironmentVariable(" LOG_LEVEL ", " Basic ", " level ")); + environment.getVariables().add(new EmbeddedEnvironmentVariable(" ", "ignored", "ignored")); + environment.getMandatoryVariables().add(null); + + EmbeddedEnvironmentValidator.normalize(environment); + + assertEquals("dev", environment.getName()); + assertNull(environment.getDescription()); + assertEquals(1, environment.getVariables().size()); + assertEquals("LOG_LEVEL", environment.getVariables().get(0).getName()); + assertEquals("Basic", environment.getVariables().get(0).getDefaultValue()); + assertEquals("level", environment.getVariables().get(0).getDescription()); + assertTrue(environment.getMandatoryVariables().isEmpty()); + } + + @Test + void duplicateVariableNameSpansTheThreeLists() { + EmbeddedEnvironment environment = environmentNamed("dev"); + environment.getVariables().add(new EmbeddedEnvironmentVariable("DB_HOST", "localhost", null)); + environment + .getSecretVariables() + .add(new EmbeddedEnvironmentVariable(" DB_HOST ", "change-to-your-password", null)); + + assertEquals("DB_HOST", EmbeddedEnvironmentValidator.duplicateVariableName(environment)); + } + + @Test + void duplicateVariableNameInsideOneList() { + EmbeddedEnvironment environment = environmentNamed("dev"); + environment + .getMandatoryVariables() + .add(new EmbeddedEnvironmentVariable("DB_PORT", "5432", null)); + environment + .getMandatoryVariables() + .add(new EmbeddedEnvironmentVariable("DB_PORT", "5433", null)); + + assertEquals("DB_PORT", EmbeddedEnvironmentValidator.duplicateVariableName(environment)); + } + + @Test + void distinctVariableNamesAreAccepted() { + EmbeddedEnvironment environment = environmentNamed("dev"); + environment.getVariables().add(new EmbeddedEnvironmentVariable("LOG_LEVEL", "Basic", null)); + environment + .getMandatoryVariables() + .add(new EmbeddedEnvironmentVariable("DB_HOST", "specify the database host", null)); + environment + .getSecretVariables() + .add(new EmbeddedEnvironmentVariable("DB_PASSWORD", "change-to-your-password", null)); + + assertNull(EmbeddedEnvironmentValidator.duplicateVariableName(environment)); + assertFalse(EmbeddedEnvironmentValidator.missingName(environment)); + } + + @Test + void duplicateEnvironmentNames() { + List environments = new ArrayList<>(); + environments.add(environmentNamed("dev")); + environments.add(environmentNamed(" dev ")); + environments.add(environmentNamed("prod")); + + assertEquals("dev", EmbeddedEnvironmentValidator.duplicateEnvironmentName(environments)); + } + + @Test + void blankEnvironmentNamesAreNotDuplicates() { + List environments = new ArrayList<>(); + environments.add(environmentNamed(" ")); + environments.add(environmentNamed(null)); + + assertNull(EmbeddedEnvironmentValidator.duplicateEnvironmentName(environments)); + assertTrue(EmbeddedEnvironmentValidator.missingName(environments.get(0))); + } + + @Test + void nameTakenMatchesTrimmedNames() { + assertTrue(EmbeddedEnvironmentValidator.nameTaken(" dev ", List.of("dev", "prod"))); + assertFalse(EmbeddedEnvironmentValidator.nameTaken("qa", List.of("dev"))); + assertFalse(EmbeddedEnvironmentValidator.nameTaken(" ", List.of("dev"))); + } + + @Test + void copyDoesNotShareVariableLists() { + EmbeddedEnvironment environment = environmentNamed("dev"); + environment + .getSecretVariables() + .add(new EmbeddedEnvironmentVariable("TOKEN", "change-to-your-token", "API token")); + + EmbeddedEnvironment copy = environment.copy(); + copy.getSecretVariables().get(0).setDefaultValue("other"); + + assertEquals("change-to-your-token", environment.getSecretVariables().get(0).getDefaultValue()); + } + + private static EmbeddedEnvironment environmentNamed(String name) { + EmbeddedEnvironment environment = new EmbeddedEnvironment(); + environment.setName(name); + return environment; + } +} diff --git a/plugins/misc/projects/src/test/java/org/apache/hop/projects/project/ProjectTest.java b/plugins/misc/projects/src/test/java/org/apache/hop/projects/project/ProjectTest.java index 1653b8beecb..6655157c502 100644 --- a/plugins/misc/projects/src/test/java/org/apache/hop/projects/project/ProjectTest.java +++ b/plugins/misc/projects/src/test/java/org/apache/hop/projects/project/ProjectTest.java @@ -38,6 +38,8 @@ import org.apache.hop.core.variables.Variables; import org.apache.hop.projects.config.ProjectsConfig; import org.apache.hop.projects.config.ProjectsConfigSingleton; +import org.apache.hop.projects.environment.EmbeddedEnvironment; +import org.apache.hop.projects.environment.EmbeddedEnvironmentVariable; import org.apache.hop.projects.util.Defaults; import org.apache.hop.projects.util.ProjectsUtil; import org.junit.jupiter.api.AfterEach; @@ -92,6 +94,81 @@ public void testEnforcingExecutionInHomeSerialization() throws Exception { } } + @Test + public void testEmbeddedEnvironmentsRoundTrip() throws Exception { + File tempFile = Files.createTempFile("project-config-environments", ".json").toFile(); + tempFile.deleteOnExit(); + + try { + Project project = new Project(tempFile.getAbsolutePath()); + assertTrue(project.getEmbeddedEnvironments().isEmpty()); + project.saveToFile(); + + String emptyJson = Files.readString(tempFile.toPath()); + assertFalse(emptyJson.contains("embeddedEnvironments")); + + Project emptyRead = new Project(tempFile.getAbsolutePath()); + emptyRead.readFromFile(); + assertTrue(emptyRead.getEmbeddedEnvironments().isEmpty()); + assertNull(emptyRead.findEmbeddedEnvironment("dev")); + + EmbeddedEnvironment environment = new EmbeddedEnvironment(); + environment.setName("dev"); + environment.setDescription("Developer workstation"); + environment + .getVariables() + .add(new EmbeddedEnvironmentVariable("LOG_LEVEL", "Basic", "Hop log level")); + environment + .getMandatoryVariables() + .add( + new EmbeddedEnvironmentVariable("DB_HOST", "specify the database host", "JDBC host")); + environment + .getSecretVariables() + .add( + new EmbeddedEnvironmentVariable( + "DB_PASSWORD", "change-to-your-password", "JDBC password")); + project.getEmbeddedEnvironments().add(environment); + project.saveToFile(); + + String json = Files.readString(tempFile.toPath()); + assertTrue(json.contains("\"defaultValue\"")); + assertTrue(json.contains("\"mandatoryVariables\"")); + assertTrue(json.contains("\"secretVariables\"")); + assertFalse(json.contains("\"configurationFiles\"")); + + Project read = new Project(tempFile.getAbsolutePath()); + read.readFromFile(); + assertEquals(1, read.getEmbeddedEnvironments().size()); + EmbeddedEnvironment found = read.findEmbeddedEnvironment("dev"); + assertEquals("Developer workstation", found.getDescription()); + assertEquals("LOG_LEVEL", found.getVariables().get(0).getName()); + assertEquals("Basic", found.getVariables().get(0).getDefaultValue()); + assertEquals("DB_HOST", found.getMandatoryVariables().get(0).getName()); + assertEquals( + "specify the database host", found.getMandatoryVariables().get(0).getDefaultValue()); + assertEquals("DB_PASSWORD", found.getSecretVariables().get(0).getName()); + assertEquals("change-to-your-password", found.getSecretVariables().get(0).getDefaultValue()); + assertNull(read.findEmbeddedEnvironment("Dev")); + } finally { + tempFile.delete(); + } + } + + @Test + public void testMissingEmbeddedEnvironmentsKeyLoadsEmpty() throws Exception { + tempRoot = Files.createTempDirectory("hop-project-no-environments"); + writeMinimalConfig(tempRoot, null); + + ProjectConfig projectConfig = + new ProjectConfig( + "no-environments", tempRoot.toString(), ProjectsConfig.DEFAULT_PROJECT_CONFIG_FILENAME); + registerProject(projectConfig); + + Project project = projectConfig.loadProject(new Variables()); + assertTrue(project.getEmbeddedEnvironments().isEmpty()); + assertNull(project.findEmbeddedEnvironment("dev")); + } + @Test public void testAutoExportMetadataSerialization() throws Exception { File tempFile = Files.createTempFile("project-config-auto-export", ".json").toFile(); From 9f9fadcc99f63975c039818967b184a85194cd9f Mon Sep 17 00:00:00 2001 From: mattcasters Date: Wed, 7 Oct 2026 11:49:41 +0200 Subject: [PATCH 2/4] Issue #4193 : Create local lifecycle environments from embedded definitions Link a lifecycle environment to an embedded definition by name. Enabling it applies the placeholder defaults, then the configuration file, and a project variable still wins. Adding a project that already defines environments can create those environments on this computer. Only mandatory variables and secrets are written, and an existing environment or configuration file is left as it is. --- .../pages/projects/projects-environments.adoc | 21 +- .../EmbeddedEnvironmentMaterializer.java | 243 ++++++++++ .../environment/LifecycleEnvironment.java | 25 ++ .../LifecycleEnvironmentDialog.java | 98 +++- .../gui/CloneFromVersionControlDialog.java | 1 + .../hop/projects/gui/ProjectsGuiPlugin.java | 142 +++++- .../apache/hop/projects/project/Project.java | 43 ++ .../messages/messages_en_US.properties | 3 + .../gui/messages/messages_en_US.properties | 10 + .../EmbeddedEnvironmentMaterializerTest.java | 420 ++++++++++++++++++ 10 files changed, 1001 insertions(+), 5 deletions(-) create mode 100644 plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentMaterializer.java create mode 100644 plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentMaterializerTest.java diff --git a/docs/hop-user-manual/modules/ROOT/pages/projects/projects-environments.adoc b/docs/hop-user-manual/modules/ROOT/pages/projects/projects-environments.adoc index 47a2564b251..9dbd6ad55ea 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/projects/projects-environments.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/projects/projects-environments.adoc @@ -130,7 +130,25 @@ Do not put a real password, host, or path in the project file. *Add*, *Edit...*, and *Delete* change the list. *Edit...* sets the name, the description, and the three variable lists. -After creating a project the user interface will switch to it and ask if you want to create an environment. +When a project is added from a folder or from Git and `project-config.json` already defines environments, Hop asks whether to create those lifecycle environments on this computer. +A project with no embedded environments still asks the generic question about creating one environment, except when the project is read only or was added from Git. + +Only the *Mandatory* and *Secrets* variables are written, one configuration file per environment. +Optional variables are applied when the environment is enabled and are not stored in that file. +The value written at creation is the placeholder from the project. +Replace it with the value for this computer. +A value in the configuration file replaces the placeholder. +A variable with the same name on the project still wins. + +The files are written to the folder in the project variable `+${ENVIRONMENTS_FOLDER}+` when that variable is set. +Otherwise Hop asks for a folder. +Keep that folder outside the project so the values are not checked in. +A typical place is `+${HOP_CONFIG_FOLDER}/environments/+` followed by the project name. +Set `+${ENVIRONMENTS_FOLDER}+` on the project to reuse the same folder the next time the project is added. +An environment that already exists on this computer is left unchanged. +An existing configuration file is not overwritten. The environment is still registered when the name is free. + +In the environment properties dialog, *Embedded environment* links the environment to one of these definitions by name. == Create an environment @@ -159,6 +177,7 @@ image:hop-gui/environment/environment-dialog-general-tab.png[Environment Propert * Common build * ...|No|No| |Project|The project to which this environment belongs|No|No|The last created project +|Embedded environment|The environment definition in the project to inherit. Required values and secrets stay in the configuration files. The other variables come from the project definition|No|No|(none) |Canvas text|Large text drawn in the top-right of pipeline and workflow canvases when this environment is active|Yes|No| |=== diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentMaterializer.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentMaterializer.java new file mode 100644 index 00000000000..1864a5a83ba --- /dev/null +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentMaterializer.java @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.projects.environment; + +import java.util.ArrayList; +import java.util.List; +import lombok.Getter; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.vfs2.FileObject; +import org.apache.hop.core.Const; +import org.apache.hop.core.config.DescribedVariablesConfigFile; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.variables.DescribedVariable; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.vfs.HopVfs; +import org.apache.hop.projects.config.ProjectsConfig; +import org.apache.hop.projects.project.Project; + +/** + * Creates local lifecycle environments from the embedded definitions in a project. + * + *

The configuration file for an environment receives mandatory variables and secrets only. + * Optional variables stay in the project and are applied when the environment is enabled. + */ +public final class EmbeddedEnvironmentMaterializer { + + /** Project variable that points at the folder for local environment configuration files. */ + public static final String VARIABLE_ENVIRONMENTS_FOLDER = "ENVIRONMENTS_FOLDER"; + + private EmbeddedEnvironmentMaterializer() {} + + /** + * Resolve {@link #VARIABLE_ENVIRONMENTS_FOLDER}. A missing or blank value is left unresolved. + * + * @param variables variable space that already contains the project variables + * @return the folder, or null when the variable is missing or blank + */ + public static String resolveFolder(IVariables variables) { + if (variables == null) { + return null; + } + String raw = variables.getVariable(VARIABLE_ENVIRONMENTS_FOLDER); + if (StringUtils.isBlank(raw)) { + return null; + } + return StringUtils.trimToNull(variables.resolve(raw)); + } + + /** + * @param projectName project the folder is for + * @return an unresolved suggestion outside the project, using {@code ${HOP_CONFIG_FOLDER}} + */ + public static String suggestedConfigFolder(String projectName) { + String safe = configFileName(StringUtils.defaultIfBlank(projectName, "project")); + safe = safe.substring(0, safe.length() - ".json".length()); + return "${HOP_CONFIG_FOLDER}/environments/" + safe; + } + + /** + * @param environmentName environment name + * @return a single path segment ending in {@code .json} + */ + public static String configFileName(String environmentName) { + String sanitized = StringUtils.defaultString(environmentName).trim(); + sanitized = sanitized.replaceAll("[\\\\/]+", "_"); + sanitized = sanitized.replaceAll("[^A-Za-z0-9._-]", "_"); + // "../dev" is ".._dev" after the slash replacement. Collapse that whole prefix. + sanitized = sanitized.replaceAll("^[._]+", "_"); + if (StringUtils.isBlank(sanitized.replace("_", ""))) { + sanitized = "environment"; + } + return sanitized + ".json"; + } + + /** + * @param folder folder that will hold the file + * @param environmentName environment name + * @return path of {@code /.json} + * @throws HopException when the folder cannot be resolved + */ + public static String configFilePath(String folder, String environmentName) throws HopException { + String fileName = configFileName(environmentName); + try (FileObject parent = HopVfs.getFileObject(folder)) { + FileObject file = parent.resolveFile(fileName); + String scheme = file.getName().getScheme(); + if (scheme == null || "file".equalsIgnoreCase(scheme)) { + return file.getName().getPath(); + } + return file.getName().getURI(); + } catch (Exception e) { + throw new HopException( + "Error building a configuration file path in folder '" + folder + "'", e); + } + } + + /** + * @param folder candidate folder + * @param projectHome project home folder + * @return true when {@code folder} is the project home or a folder inside it + */ + public static boolean isInsideProjectHome(String folder, String projectHome) { + if (StringUtils.isBlank(folder) || StringUtils.isBlank(projectHome)) { + return false; + } + try (FileObject home = HopVfs.getFileObject(projectHome); + FileObject target = HopVfs.getFileObject(folder)) { + String homePath = withTrailingSlash(home.getName().getPath()); + String targetPath = withTrailingSlash(target.getName().getPath()); + return targetPath.startsWith(homePath); + } catch (Exception e) { + return false; + } + } + + /** + * Mandatory variables and secrets, in that order. Optional variables are not included. + * + * @param environment embedded definition + * @return variables to store in the local configuration file + */ + public static List variablesToStore(EmbeddedEnvironment environment) { + List stored = new ArrayList<>(); + addVariables(stored, environment == null ? null : environment.getMandatoryVariables()); + addVariables(stored, environment == null ? null : environment.getSecretVariables()); + return stored; + } + + /** + * Create a lifecycle environment for every embedded definition that is not already registered. + * Existing configuration files are left as they are. Nothing is saved to {@code hop-config.json}; + * the caller registers {@link MaterializeResult#getCreated()}. + * + * @param config environments already on this computer + * @param project project that holds the definitions + * @param projectName project the new environments belong to + * @param folder folder for the configuration files + * @return what was created and what was left alone + * @throws HopException when a configuration file cannot be written + */ + public static MaterializeResult materialize( + ProjectsConfig config, Project project, String projectName, String folder) + throws HopException { + MaterializeResult result = new MaterializeResult(); + if (project == null || project.getEmbeddedEnvironments() == null) { + return result; + } + if (StringUtils.isBlank(folder)) { + throw new HopException("A folder is required to create environment configuration files"); + } + for (EmbeddedEnvironment embedded : project.getEmbeddedEnvironments()) { + if (embedded == null || StringUtils.isBlank(embedded.getName())) { + continue; + } + if (config != null && config.findEnvironment(embedded.getName()) != null) { + result.skippedExistingNames.add(embedded.getName()); + continue; + } + String path = configFilePath(folder, embedded.getName()); + if (!writeConfigFileIfMissing(path, embedded)) { + result.keptExistingFiles.add(embedded.getName()); + } + LifecycleEnvironment environment = + new LifecycleEnvironment( + embedded.getName(), "", projectName, new ArrayList<>(List.of(path))); + environment.setEmbeddedEnvironmentName(embedded.getName()); + result.created.add(environment); + } + return result; + } + + /** + * @param path configuration file path + * @param embedded definition whose mandatory variables and secrets are written + * @return true when a new file was written, false when a file was already there + * @throws HopException when the file cannot be written + */ + public static boolean writeConfigFileIfMissing(String path, EmbeddedEnvironment embedded) + throws HopException { + try (FileObject file = HopVfs.getFileObject(path)) { + if (file.exists()) { + return false; + } + if (file.getParent() != null && !file.getParent().exists()) { + file.getParent().createFolder(); + } + } catch (Exception e) { + throw new HopException("Error checking configuration file '" + path + "'", e); + } + DescribedVariablesConfigFile configFile = new DescribedVariablesConfigFile(path); + configFile.setDescription(embedded.getDescription()); + configFile.setDescribedVariables(variablesToStore(embedded)); + configFile.saveToFile(); + return true; + } + + private static void addVariables( + List stored, List source) { + if (source == null) { + return; + } + for (EmbeddedEnvironmentVariable variable : source) { + if (variable == null || StringUtils.isBlank(variable.getName())) { + continue; + } + stored.add( + new DescribedVariable( + variable.getName().trim(), + Const.NVL(variable.getDefaultValue(), ""), + Const.NVL(variable.getDescription(), ""))); + } + } + + private static String withTrailingSlash(String path) { + String normalized = path.replace('\\', '/'); + if (!normalized.endsWith("/")) { + normalized = normalized + "/"; + } + return normalized; + } + + /** Environments created by {@link #materialize} and the ones that were left alone. */ + @Getter + public static final class MaterializeResult { + private final List created = new ArrayList<>(); + private final List skippedExistingNames = new ArrayList<>(); + private final List keptExistingFiles = new ArrayList<>(); + } +} diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/LifecycleEnvironment.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/LifecycleEnvironment.java index 6c0eba8af87..2b78b17d3d3 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/LifecycleEnvironment.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/LifecycleEnvironment.java @@ -17,6 +17,7 @@ package org.apache.hop.projects.environment; +import com.fasterxml.jackson.annotation.JsonInclude; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -43,6 +44,13 @@ public class LifecycleEnvironment implements IAttributes { private List configurationFiles; + /** + * Name of the embedded environment in the project definition. Null when this environment is not + * linked. Real values stay in the configuration files. + */ + @JsonInclude(JsonInclude.Include.NON_NULL) + private String embeddedEnvironmentName; + /** Group → (key → value); see {@link IAttributes}. */ private Map> attributesMap; @@ -66,6 +74,7 @@ public LifecycleEnvironment(LifecycleEnvironment env) { this.projectName = env.projectName; this.canvasText = env.canvasText; this.configurationFiles = new ArrayList<>(env.configurationFiles); + this.embeddedEnvironmentName = env.embeddedEnvironmentName; this.attributesMap = deepCopyAttributes(env.attributesMap); } @@ -183,6 +192,22 @@ public void setConfigurationFiles(List configurationFiles) { this.configurationFiles = configurationFiles; } + /** + * Gets embeddedEnvironmentName + * + * @return value of embeddedEnvironmentName + */ + public String getEmbeddedEnvironmentName() { + return embeddedEnvironmentName; + } + + /** + * @param embeddedEnvironmentName The embedded environment name to set. Null clears the link. + */ + public void setEmbeddedEnvironmentName(String embeddedEnvironmentName) { + this.embeddedEnvironmentName = embeddedEnvironmentName; + } + @Override public void setAttributesMap(Map> attributesMap) { this.attributesMap = attributesMap != null ? attributesMap : new HashMap<>(); diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/LifecycleEnvironmentDialog.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/LifecycleEnvironmentDialog.java index ed6c69c8a2c..ff27dc674d2 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/LifecycleEnvironmentDialog.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/LifecycleEnvironmentDialog.java @@ -36,6 +36,7 @@ import org.apache.hop.i18n.BaseMessages; import org.apache.hop.projects.config.ProjectsConfig; import org.apache.hop.projects.config.ProjectsConfigSingleton; +import org.apache.hop.projects.project.Project; import org.apache.hop.projects.project.ProjectConfig; import org.apache.hop.projects.util.Defaults; import org.apache.hop.projects.util.PathVariableReplacer; @@ -95,6 +96,7 @@ public class LifecycleEnvironmentDialog extends Dialog { private TextVar wName; private Combo wPurpose; private Combo wProject; + private Combo wEmbeddedEnvironment; private Text wCanvasText; private TableView wConfigFiles; @@ -305,8 +307,30 @@ private void createGeneralTab(CTabFolder folder, int margin) { e -> { needingEnvironmentRefresh = true; updateSuggestedName(); + refreshEmbeddedEnvironmentItems(); }); + Label wlEmbedded = new Label(comp, SWT.RIGHT); + PropsUi.setLook(wlEmbedded); + wlEmbedded.setText( + BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.Label.EmbeddedEnvironment")); + wlEmbedded.setToolTipText( + BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.ToolTip.EmbeddedEnvironment")); + FormData fdlEmbedded = new FormData(); + fdlEmbedded.left = new FormAttachment(0, 0); + fdlEmbedded.right = new FormAttachment(middle, 0); + fdlEmbedded.top = new FormAttachment(wProject, margin); + wlEmbedded.setLayoutData(fdlEmbedded); + wEmbeddedEnvironment = new Combo(comp, SWT.DROP_DOWN | SWT.READ_ONLY); + PropsUi.setLook(wEmbeddedEnvironment); + wEmbeddedEnvironment.setToolTipText( + BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.ToolTip.EmbeddedEnvironment")); + FormData fdEmbedded = new FormData(); + fdEmbedded.left = new FormAttachment(middle, margin); + fdEmbedded.right = new FormAttachment(100, 0); + fdEmbedded.top = new FormAttachment(wlEmbedded, 0, SWT.CENTER); + wEmbeddedEnvironment.setLayoutData(fdEmbedded); + Label wlCanvasText = new Label(comp, SWT.RIGHT); PropsUi.setLook(wlCanvasText); wlCanvasText.setText( @@ -314,7 +338,7 @@ private void createGeneralTab(CTabFolder folder, int margin) { FormData fdlCanvasText = new FormData(); fdlCanvasText.left = new FormAttachment(0, 0); fdlCanvasText.right = new FormAttachment(middle, 0); - fdlCanvasText.top = new FormAttachment(wProject, margin); + fdlCanvasText.top = new FormAttachment(wEmbeddedEnvironment, margin); wlCanvasText.setLayoutData(fdlCanvasText); wCanvasText = new Text(comp, SWT.SINGLE | SWT.BORDER | SWT.LEFT); PropsUi.setLook(wCanvasText); @@ -1042,6 +1066,71 @@ private String projectHomeOf(String projectName) { } } + private String noneEmbeddedEnvironment() { + return BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.EmbeddedEnvironment.None"); + } + + /** + * Fill the embedded-environment combo from the selected project. A name that is no longer defined + * stays in the list so the broken link remains visible. + */ + private void refreshEmbeddedEnvironmentItems() { + if (wEmbeddedEnvironment == null || wEmbeddedEnvironment.isDisposed()) { + return; + } + String none = noneEmbeddedEnvironment(); + String current = wEmbeddedEnvironment.getText(); + List items = new ArrayList<>(); + items.add(none); + items.addAll(embeddedEnvironmentNames(wProject == null ? null : wProject.getText())); + if (StringUtils.isNotEmpty(current) && !none.equals(current) && !items.contains(current)) { + items.add(current); + } + wEmbeddedEnvironment.setItems(items.toArray(new String[0])); + wEmbeddedEnvironment.setText(StringUtils.isEmpty(current) ? none : current); + } + + private void selectEmbeddedEnvironment(String name) { + refreshEmbeddedEnvironmentItems(); + if (StringUtils.isEmpty(name)) { + wEmbeddedEnvironment.setText(noneEmbeddedEnvironment()); + return; + } + List items = new ArrayList<>(List.of(wEmbeddedEnvironment.getItems())); + if (!items.contains(name)) { + items.add(name); + wEmbeddedEnvironment.setItems(items.toArray(new String[0])); + } + wEmbeddedEnvironment.setText(name); + } + + private List embeddedEnvironmentNames(String projectName) { + List names = new ArrayList<>(); + if (StringUtils.isEmpty(projectName)) { + return names; + } + try { + ProjectsConfig config = ProjectsConfigSingleton.getConfig(); + ProjectConfig projectConfig = config == null ? null : config.findProjectConfig(projectName); + if (projectConfig == null) { + return names; + } + Project project = projectConfig.loadProject(variables); + if (project.getEmbeddedEnvironments() == null) { + return names; + } + for (EmbeddedEnvironment embedded : project.getEmbeddedEnvironments()) { + if (embedded != null && StringUtils.isNotEmpty(embedded.getName())) { + names.add(embedded.getName()); + } + } + } catch (Exception e) { + LogChannel.UI.logError( + "Could not load embedded environments for project '" + projectName + "'", e); + } + return names; + } + private void getData() { ProjectsConfig config = ProjectsConfigSingleton.getConfig(); @@ -1080,6 +1169,7 @@ private void getData() { wPurpose.setText(Const.NVL(environment.getPurpose(), "")); wProject.setText(Const.NVL(environment.getProjectName(), "")); wCanvasText.setText(Const.NVL(environment.getCanvasText(), "")); + selectEmbeddedEnvironment(environment.getEmbeddedEnvironmentName()); if (StringUtils.isNotEmpty(environment.getName())) { // Provided name (edit or rare pre-fill): do not auto-overwrite. @@ -1167,6 +1257,12 @@ private void getInfo(LifecycleEnvironment env) { env.setPurpose(wPurpose.getText()); env.setProjectName(wProject.getText()); env.setCanvasText(wCanvasText.getText()); + String embeddedName = wEmbeddedEnvironment.getText(); + if (StringUtils.isEmpty(embeddedName) || noneEmbeddedEnvironment().equals(embeddedName)) { + env.setEmbeddedEnvironmentName(null); + } else { + env.setEmbeddedEnvironmentName(embeddedName); + } env.getConfigurationFiles().clear(); for (TableItem item : wConfigFiles.getNonEmptyItems()) { diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/CloneFromVersionControlDialog.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/CloneFromVersionControlDialog.java index 0cb71ee43f4..b99137e885a 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/CloneFromVersionControlDialog.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/CloneFromVersionControlDialog.java @@ -340,6 +340,7 @@ private void ok() { HopGui hopGui = HopGui.getInstance(); ProjectsGuiPlugin.updateProjectToolItem(projectName); ProjectsGuiPlugin.enableHopGuiProject(projectName, project, null); + ProjectsGuiPlugin.offerEmbeddedEnvironments(shell, project, projectConfig, variables); returnValue = projectName; dispose(); diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java index 9db4114a1d5..f4bedb6a19f 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java @@ -70,6 +70,8 @@ import org.apache.hop.projects.config.ProjectsConfig; import org.apache.hop.projects.config.ProjectsConfigOptionPlugin; import org.apache.hop.projects.config.ProjectsConfigSingleton; +import org.apache.hop.projects.environment.EmbeddedEnvironment; +import org.apache.hop.projects.environment.EmbeddedEnvironmentMaterializer; import org.apache.hop.projects.environment.LifecycleEnvironment; import org.apache.hop.projects.environment.LifecycleEnvironmentDialog; import org.apache.hop.projects.project.Project; @@ -1367,9 +1369,12 @@ private void addProjectFromExistingSourcesInternal() { wrcSerializer.save(local); } } + } - // Ask to put the project in a lifecycle environment - // + // Embedded definitions replace the generic question. A new project without them keeps it. + // + if (!offerEmbeddedEnvironments(hopGui.getActiveShell(), project, projectConfig, variables) + && !projectConfig.isReadOnly()) { MessageBox box = new MessageBox(HopGui.getInstance().getShell(), SWT.YES | SWT.NO | SWT.ICON_QUESTION); box.setText(BaseMessages.getString(PKG, "ProjectGuiPlugin.Lifecycle.Dialog.Header")); @@ -1379,7 +1384,6 @@ private void addProjectFromExistingSourcesInternal() { + BaseMessages.getString(PKG, "ProjectGuiPlugin.Lifecycle.Dialog.Message2")); int answer = box.open(); if ((answer & SWT.YES) != 0) { - addNewEnvironment(); } } @@ -1602,6 +1606,138 @@ public void selectEnvironment(String environmentName) { } } + /** + * Ask whether to create local lifecycle environments from the definitions stored in the project. + * Required values and secrets are written to one configuration file per environment. The project + * file is not changed. + * + * @return true when the project defines embedded environments, including when the user declines + * or the folder dialog is cancelled. The generic environment question must not follow. + */ + public static boolean offerEmbeddedEnvironments( + Shell shell, Project project, ProjectConfig projectConfig, IVariables variables) { + List names = embeddedEnvironmentNames(project); + if (names.isEmpty()) { + return false; + } + + String projectName = projectConfig == null ? "" : Const.NVL(projectConfig.getProjectName(), ""); + String folder = EmbeddedEnvironmentMaterializer.resolveFolder(variables); + String message = + BaseMessages.getString( + PKG, + "ProjectGuiPlugin.EmbeddedEnvironments.Question.Message", + projectName, + String.join(", ", names)); + if (folder != null) { + message = + message + + Const.CR + + Const.CR + + BaseMessages.getString( + PKG, "ProjectGuiPlugin.EmbeddedEnvironments.Question.Folder", folder); + } + + MessageBox question = new MessageBox(shell, SWT.YES | SWT.NO | SWT.ICON_QUESTION); + question.setText( + BaseMessages.getString(PKG, "ProjectGuiPlugin.EmbeddedEnvironments.Question.Header")); + question.setMessage(message); + if ((question.open() & SWT.YES) == 0) { + return true; + } + + if (folder == null) { + folder = chooseEnvironmentFolder(shell, projectName, variables); + if (folder == null) { + return true; + } + } + + String projectHome = null; + if (projectConfig != null && variables != null) { + projectHome = variables.resolve(projectConfig.getProjectHome()); + } + if (EmbeddedEnvironmentMaterializer.isInsideProjectHome(folder, projectHome)) { + MessageBox warning = new MessageBox(shell, SWT.YES | SWT.NO | SWT.ICON_WARNING); + warning.setText( + BaseMessages.getString( + PKG, "ProjectGuiPlugin.EmbeddedEnvironments.InsideProject.Header")); + warning.setMessage( + BaseMessages.getString( + PKG, "ProjectGuiPlugin.EmbeddedEnvironments.InsideProject.Message")); + if ((warning.open() & SWT.YES) == 0) { + return true; + } + } + + try { + ProjectsConfig config = ProjectsConfigSingleton.getConfig(); + EmbeddedEnvironmentMaterializer.MaterializeResult result = + EmbeddedEnvironmentMaterializer.materialize(config, project, projectName, folder); + for (LifecycleEnvironment created : result.getCreated()) { + config.addEnvironment(created); + } + if (!result.getCreated().isEmpty()) { + ProjectsConfigSingleton.saveConfig(); + } + if (!result.getSkippedExistingNames().isEmpty()) { + MessageBox skipped = new MessageBox(shell, SWT.OK | SWT.ICON_INFORMATION); + skipped.setText( + BaseMessages.getString(PKG, "ProjectGuiPlugin.EmbeddedEnvironments.Skipped.Header")); + skipped.setMessage( + BaseMessages.getString( + PKG, + "ProjectGuiPlugin.EmbeddedEnvironments.Skipped.Message", + String.join(Const.CR, result.getSkippedExistingNames()))); + skipped.open(); + } + if (result.getCreated().size() == 1) { + LifecycleEnvironment created = result.getCreated().get(0); + enableHopGuiProject(projectName, project, created); + updateEnvironmentToolItem(created.getName()); + } + } catch (Exception e) { + new ErrorDialog( + shell, + BaseMessages.getString(PKG, "ProjectGuiPlugin.EmbeddedEnvironments.Error.Header"), + BaseMessages.getString(PKG, "ProjectGuiPlugin.EmbeddedEnvironments.Error.Message"), + e); + } + return true; + } + + private static String chooseEnvironmentFolder( + Shell shell, String projectName, IVariables variables) { + String suggestion = EmbeddedEnvironmentMaterializer.suggestedConfigFolder(projectName); + String resolved = variables == null ? suggestion : variables.resolve(suggestion); + if (StringUtils.isBlank(resolved) || resolved.contains("${")) { + String safe = + EmbeddedEnvironmentMaterializer.configFileName( + StringUtils.defaultIfBlank(projectName, "project")); + safe = safe.substring(0, safe.length() - ".json".length()); + resolved = Const.HOP_CONFIG_FOLDER + "/environments/" + safe; + } + return BaseDialog.presentDirectoryDialog( + shell, + resolved, + BaseMessages.getString(PKG, "ProjectGuiPlugin.EmbeddedEnvironments.Folder.Message"), + variables); + } + + /** Names of the embedded environments that are safe to show and to create. */ + private static List embeddedEnvironmentNames(Project project) { + List names = new ArrayList<>(); + if (project == null || project.getEmbeddedEnvironments() == null) { + return names; + } + for (EmbeddedEnvironment environment : project.getEmbeddedEnvironments()) { + if (environment != null && StringUtils.isNotBlank(environment.getName())) { + names.add(environment.getName().trim()); + } + } + return names; + } + @GuiMenuElement( root = ID_CONTEXT_MENU_ENVIRONMENT, parentId = ID_CONTEXT_MENU_ENVIRONMENT, diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/Project.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/Project.java index 0e4a67f0279..9a43c55573c 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/Project.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/Project.java @@ -65,6 +65,8 @@ import org.apache.hop.projects.config.ProjectsConfig; import org.apache.hop.projects.config.ProjectsConfigSingleton; import org.apache.hop.projects.environment.EmbeddedEnvironment; +import org.apache.hop.projects.environment.EmbeddedEnvironmentVariable; +import org.apache.hop.projects.environment.LifecycleEnvironment; import org.apache.hop.projects.util.Defaults; import org.apache.hop.projects.util.ProjectsUtil; import org.apache.hop.workflow.WorkflowMeta; @@ -303,6 +305,11 @@ public void modifyVariables( // applyProjectVariables(variables); + // Placeholder defaults from the linked embedded environment. Configuration files below replace + // them, and the final applyProjectVariables() still lets a project variable win. + // + applyEmbeddedEnvironmentDefaults(variables, environmentName); + // Apply the described variables from the various configuration files in the given order... // for (String configurationFile : configurationFiles) { @@ -366,6 +373,42 @@ public void modifyVariables( applyProjectVariables(variables); } + private void applyEmbeddedEnvironmentDefaults(IVariables variables, String environmentName) { + if (StringUtils.isEmpty(environmentName)) { + return; + } + LifecycleEnvironment lifecycleEnvironment = + ProjectsConfigSingleton.getConfig().findEnvironment(environmentName); + if (lifecycleEnvironment == null + || StringUtils.isEmpty(lifecycleEnvironment.getEmbeddedEnvironmentName())) { + return; + } + EmbeddedEnvironment embedded = + findEmbeddedEnvironment(lifecycleEnvironment.getEmbeddedEnvironmentName()); + if (embedded == null) { + LogChannel.GENERAL.logError( + "Embedded environment '" + + lifecycleEnvironment.getEmbeddedEnvironmentName() + + "' is not defined in this project. Continuing without those defaults."); + return; + } + applyEmbeddedVariables(variables, embedded.getVariables()); + applyEmbeddedVariables(variables, embedded.getMandatoryVariables()); + applyEmbeddedVariables(variables, embedded.getSecretVariables()); + } + + private static void applyEmbeddedVariables( + IVariables variables, List defined) { + if (defined == null) { + return; + } + for (EmbeddedEnvironmentVariable variable : defined) { + if (variable != null && StringUtils.isNotEmpty(variable.getName())) { + variables.setVariable(variable.getName(), Const.NVL(variable.getDefaultValue(), "")); + } + } + } + private void applyProjectVariables(IVariables variables) { for (DescribedVariable variable : getDescribedVariables()) { if (variable.getName() != null) { diff --git a/plugins/misc/projects/src/main/resources/org/apache/hop/projects/environment/messages/messages_en_US.properties b/plugins/misc/projects/src/main/resources/org/apache/hop/projects/environment/messages/messages_en_US.properties index 8c307e74e08..064111eca3d 100644 --- a/plugins/misc/projects/src/main/resources/org/apache/hop/projects/environment/messages/messages_en_US.properties +++ b/plugins/misc/projects/src/main/resources/org/apache/hop/projects/environment/messages/messages_en_US.properties @@ -50,6 +50,9 @@ LifecycleEnvironmentDialog.Group.Label.ConfigurationFiles=Configuration files: LifecycleEnvironmentDialog.Label.EnvironmentName=Name LifecycleEnvironmentDialog.Label.EnvironmentPurpose=Purpose LifecycleEnvironmentDialog.Label.ReferencedProject=Project +LifecycleEnvironmentDialog.Label.EmbeddedEnvironment=Embedded environment +LifecycleEnvironmentDialog.EmbeddedEnvironment.None=(none) +LifecycleEnvironmentDialog.ToolTip.EmbeddedEnvironment=Required values and secrets are stored in the configuration files. The other variables come from this environment definition in the project. LifecycleEnvironmentDialog.Label.CanvasText=Canvas text LifecycleEnvironmentDialog.ToolTip.CanvasText=Large text drawn in the top-right of pipeline and workflow canvases when this environment is active LifecycleEnvironmentDialog.Purpose.Text.Acceptance=Acceptance diff --git a/plugins/misc/projects/src/main/resources/org/apache/hop/projects/gui/messages/messages_en_US.properties b/plugins/misc/projects/src/main/resources/org/apache/hop/projects/gui/messages/messages_en_US.properties index 4e2e758bd0e..096ffaa76fa 100644 --- a/plugins/misc/projects/src/main/resources/org/apache/hop/projects/gui/messages/messages_en_US.properties +++ b/plugins/misc/projects/src/main/resources/org/apache/hop/projects/gui/messages/messages_en_US.properties @@ -106,6 +106,16 @@ ProjectGuiPlugin.EditProject.Error.Dialog.Message=Error editing project ''{0}'' ProjectGuiPlugin.Lifecycle.Dialog.Header=Create project lifecycle environment? ProjectGuiPlugin.Lifecycle.Dialog.Message1=If this project is part of a lifecyle then perhaps you want to add it to a lifecycle environment? ProjectGuiPlugin.Lifecycle.Dialog.Message2=With it you can manage the specific settings like hostnames and paths for the environment +ProjectGuiPlugin.EmbeddedEnvironments.Question.Header=Create lifecycle environments? +ProjectGuiPlugin.EmbeddedEnvironments.Question.Message=Project ''{0}'' defines lifecycle environments: {1}.\n\nCreate them on this computer? Required variables and secrets are written to a configuration file. Other variables keep the defaults from the project. +ProjectGuiPlugin.EmbeddedEnvironments.Question.Folder=Configuration files will be written to ''{0}''. +ProjectGuiPlugin.EmbeddedEnvironments.Folder.Message=Choose the folder for the configuration files. Set the project variable '${ENVIRONMENTS_FOLDER}' to reuse that folder the next time someone adds this project. A folder outside the project keeps secrets out of version control. +ProjectGuiPlugin.EmbeddedEnvironments.InsideProject.Header=Folder is inside the project +ProjectGuiPlugin.EmbeddedEnvironments.InsideProject.Message=This folder is inside the project home. The configuration files will hold the values you type there, so keep them out of version control.\n\nCreate the files in this folder anyway? +ProjectGuiPlugin.EmbeddedEnvironments.Skipped.Header=Some environments were not created +ProjectGuiPlugin.EmbeddedEnvironments.Skipped.Message=These environments already exist on this computer and were left unchanged:\n{0} +ProjectGuiPlugin.EmbeddedEnvironments.Error.Header=Error +ProjectGuiPlugin.EmbeddedEnvironments.Error.Message=The lifecycle environments could not be created. ProjectGuiPlugin.LocalWFRunConfig.Dialog.Header=Create local workflow run configuration? ProjectGuiPlugin.LocalWFRunConfig.Dialog.Message=Do you want to have a local workflow run configuration for this project? ProjectGuiPlugin.LocalWFRunConfigDescription.Text=Runs your workflows locally with the standard local Hop workflow engine diff --git a/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentMaterializerTest.java b/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentMaterializerTest.java new file mode 100644 index 00000000000..f30572e8e1e --- /dev/null +++ b/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentMaterializerTest.java @@ -0,0 +1,420 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.projects.environment; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Stream; +import org.apache.commons.vfs2.FileObject; +import org.apache.hop.core.config.DescribedVariablesConfigFile; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.json.HopJson; +import org.apache.hop.core.logging.HopLogStore; +import org.apache.hop.core.logging.HopLoggingEvent; +import org.apache.hop.core.logging.LogChannel; +import org.apache.hop.core.variables.DescribedVariable; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.variables.Variables; +import org.apache.hop.core.vfs.HopVfs; +import org.apache.hop.projects.config.ProjectsConfig; +import org.apache.hop.projects.config.ProjectsConfigSingleton; +import org.apache.hop.projects.environment.EmbeddedEnvironmentMaterializer.MaterializeResult; +import org.apache.hop.projects.project.Project; +import org.apache.hop.projects.project.ProjectConfig; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class EmbeddedEnvironmentMaterializerTest { + + private final List registeredEnvironments = new ArrayList<>(); + private Path tempRoot; + + @BeforeEach + void setUp() throws Exception { + HopLogStore.init(); + tempRoot = Files.createTempDirectory("hop-embedded-env"); + } + + @AfterEach + void tearDown() throws Exception { + ProjectsConfig config = ProjectsConfigSingleton.getConfig(); + for (String name : registeredEnvironments) { + config.removeEnvironment(name); + } + registeredEnvironments.clear(); + if (tempRoot != null && Files.exists(tempRoot)) { + try (Stream walk = Files.walk(tempRoot)) { + walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } + tempRoot = null; + } + } + + @Test + void configFileNameSanitizesPathCharacters() { + assertEquals("dev.json", EmbeddedEnvironmentMaterializer.configFileName("dev")); + assertEquals("_dev.json", EmbeddedEnvironmentMaterializer.configFileName("../dev")); + assertEquals("a_b.json", EmbeddedEnvironmentMaterializer.configFileName("a/b")); + assertEquals("My_Project.json", EmbeddedEnvironmentMaterializer.configFileName("My Project")); + assertEquals("_hidden.json", EmbeddedEnvironmentMaterializer.configFileName(".hidden")); + assertEquals("environment.json", EmbeddedEnvironmentMaterializer.configFileName(" ")); + } + + @Test + void suggestedConfigFolderStaysOutsideTheProject() { + assertEquals( + "${HOP_CONFIG_FOLDER}/environments/My_Project", + EmbeddedEnvironmentMaterializer.suggestedConfigFolder("My Project")); + assertEquals( + "${HOP_CONFIG_FOLDER}/environments/project", + EmbeddedEnvironmentMaterializer.suggestedConfigFolder(null)); + assertEquals( + "${HOP_CONFIG_FOLDER}/environments/_dev", + EmbeddedEnvironmentMaterializer.suggestedConfigFolder("../dev")); + } + + @Test + void resolveFolderUsesTheProjectVariable() { + assertNull(EmbeddedEnvironmentMaterializer.resolveFolder(null)); + + IVariables missing = new Variables(); + assertNull(EmbeddedEnvironmentMaterializer.resolveFolder(missing)); + + IVariables blank = new Variables(); + blank.setVariable(EmbeddedEnvironmentMaterializer.VARIABLE_ENVIRONMENTS_FOLDER, " "); + assertNull(EmbeddedEnvironmentMaterializer.resolveFolder(blank)); + + IVariables variables = new Variables(); + variables.setVariable("PROJECT_HOME", "/data/proj"); + variables.setVariable( + EmbeddedEnvironmentMaterializer.VARIABLE_ENVIRONMENTS_FOLDER, + "${PROJECT_HOME}/environments"); + assertEquals( + "/data/proj/environments", EmbeddedEnvironmentMaterializer.resolveFolder(variables)); + } + + @Test + void configFilePathDoesNotEscapeTheFolder() throws Exception { + String path = EmbeddedEnvironmentMaterializer.configFilePath(tempRoot.toString(), "../dev"); + assertTrue(path.endsWith("_dev.json")); + assertFalse(path.contains("..")); + assertTrue(path.contains(tempRoot.getFileName().toString())); + } + + @Test + void insideProjectHomeMatchesTheHomeAndItsChildren() throws Exception { + Path home = tempRoot.resolve("proj"); + Path extra = tempRoot.resolve("proj-extra"); + Path child = home.resolve("environments"); + Files.createDirectories(child); + Files.createDirectories(extra); + + assertTrue( + EmbeddedEnvironmentMaterializer.isInsideProjectHome(home.toString(), home.toString())); + assertTrue( + EmbeddedEnvironmentMaterializer.isInsideProjectHome(child.toString(), home.toString())); + assertFalse( + EmbeddedEnvironmentMaterializer.isInsideProjectHome(extra.toString(), home.toString())); + assertFalse(EmbeddedEnvironmentMaterializer.isInsideProjectHome("", home.toString())); + assertFalse(EmbeddedEnvironmentMaterializer.isInsideProjectHome(child.toString(), " ")); + } + + @Test + void variablesToStoreKeepsMandatoryAndSecretsOnly() { + EmbeddedEnvironment environment = sampleEnvironment("dev"); + List stored = EmbeddedEnvironmentMaterializer.variablesToStore(environment); + + assertEquals(2, stored.size()); + assertEquals("DB_HOST", stored.get(0).getName()); + assertEquals("specify the database host", stored.get(0).getValue()); + assertEquals("JDBC host", stored.get(0).getDescription()); + assertEquals("DB_PASSWORD", stored.get(1).getName()); + assertEquals("change-to-your-password", stored.get(1).getValue()); + assertEquals("JDBC password", stored.get(1).getDescription()); + assertTrue(EmbeddedEnvironmentMaterializer.variablesToStore(null).isEmpty()); + } + + @Test + void materializeWritesMandatoryAndSecrets() throws Exception { + EmbeddedEnvironment environment = sampleEnvironment("dev"); + environment + .getMandatoryVariables() + .add(new EmbeddedEnvironmentVariable("EMPTY_DEFAULT", null, null)); + Project project = new Project(); + project.getEmbeddedEnvironments().add(environment); + project.getEmbeddedEnvironments().add(new EmbeddedEnvironment()); + project.getEmbeddedEnvironments().add(null); + + MaterializeResult result = + EmbeddedEnvironmentMaterializer.materialize( + new ProjectsConfig(), project, "warehouse", tempRoot.toString()); + + assertEquals(1, result.getCreated().size()); + assertTrue(result.getSkippedExistingNames().isEmpty()); + assertTrue(result.getKeptExistingFiles().isEmpty()); + LifecycleEnvironment created = result.getCreated().get(0); + assertEquals("dev", created.getName()); + assertEquals("", created.getPurpose()); + assertEquals("warehouse", created.getProjectName()); + assertEquals("dev", created.getEmbeddedEnvironmentName()); + assertEquals(1, created.getConfigurationFiles().size()); + + String path = created.getConfigurationFiles().get(0); + assertTrue(fileExists(path)); + String json = Files.readString(Path.of(path)); + assertFalse(json.contains("OPTIONAL_VAR")); + assertFalse(json.contains("configurationFiles")); + + DescribedVariablesConfigFile configFile = new DescribedVariablesConfigFile(path); + configFile.readFromFile(); + assertEquals("Developer workstation", configFile.getDescription()); + List stored = configFile.getDescribedVariables(); + assertEquals(3, stored.size()); + assertEquals("DB_HOST", stored.get(0).getName()); + assertEquals("EMPTY_DEFAULT", stored.get(1).getName()); + assertEquals("", stored.get(1).getValue()); + assertEquals("", stored.get(1).getDescription()); + assertEquals("DB_PASSWORD", stored.get(2).getName()); + assertEquals("change-to-your-password", stored.get(2).getValue()); + + assertTrue( + EmbeddedEnvironmentMaterializer.materialize(null, null, "warehouse", tempRoot.toString()) + .getCreated() + .isEmpty()); + } + + @Test + void materializeSkipsAnEnvironmentThatAlreadyExists() throws Exception { + ProjectsConfig config = new ProjectsConfig(); + config.addEnvironment(new LifecycleEnvironment("dev", "keep", "other", new ArrayList<>())); + Project project = new Project(); + project.getEmbeddedEnvironments().add(sampleEnvironment("dev")); + + MaterializeResult result = + EmbeddedEnvironmentMaterializer.materialize( + config, project, "warehouse", tempRoot.toString()); + + assertTrue(result.getCreated().isEmpty()); + assertEquals(List.of("dev"), result.getSkippedExistingNames()); + assertEquals("keep", config.findEnvironment("dev").getPurpose()); + assertFalse( + fileExists(EmbeddedEnvironmentMaterializer.configFilePath(tempRoot.toString(), "dev"))); + } + + @Test + void materializeKeepsAnExistingConfigurationFile() throws Exception { + String path = EmbeddedEnvironmentMaterializer.configFilePath(tempRoot.toString(), "dev"); + DescribedVariablesConfigFile existing = new DescribedVariablesConfigFile(path); + existing.setDescribedVariables( + List.of(new DescribedVariable("DB_PASSWORD", "real-password", "JDBC password"))); + existing.saveToFile(); + + Project project = new Project(); + project.getEmbeddedEnvironments().add(sampleEnvironment("dev")); + MaterializeResult result = + EmbeddedEnvironmentMaterializer.materialize( + new ProjectsConfig(), project, "warehouse", tempRoot.toString()); + + assertEquals(List.of("dev"), result.getKeptExistingFiles()); + assertEquals(1, result.getCreated().size()); + assertEquals(path, result.getCreated().get(0).getConfigurationFiles().get(0)); + + DescribedVariablesConfigFile reread = new DescribedVariablesConfigFile(path); + reread.readFromFile(); + assertEquals("real-password", reread.findDescribedVariableValue("DB_PASSWORD")); + assertFalse(Files.readString(Path.of(path)).contains("change-to-your-password")); + } + + @Test + void materializeRequiresAFolder() { + Project project = new Project(); + project.getEmbeddedEnvironments().add(sampleEnvironment("dev")); + assertThrows( + HopException.class, + () -> + EmbeddedEnvironmentMaterializer.materialize( + new ProjectsConfig(), project, "warehouse", " ")); + } + + @Test + void lifecycleEnvironmentLinkIsCopiedAndOmittedWhenNull() throws Exception { + LifecycleEnvironment environment = + new LifecycleEnvironment("dev", "", "warehouse", new ArrayList<>()); + String withoutLink = HopJson.newMapper().writeValueAsString(environment); + assertFalse(withoutLink.contains("embeddedEnvironmentName")); + + environment.setEmbeddedEnvironmentName(""); + String emptyLink = HopJson.newMapper().writeValueAsString(environment); + assertTrue(emptyLink.contains("\"embeddedEnvironmentName\":\"\"")); + + environment.setEmbeddedEnvironmentName("dev"); + String withLink = HopJson.newMapper().writeValueAsString(environment); + assertTrue(withLink.contains("\"embeddedEnvironmentName\":\"dev\"")); + + LifecycleEnvironment copy = new LifecycleEnvironment(environment); + assertEquals("dev", copy.getEmbeddedEnvironmentName()); + copy.setEmbeddedEnvironmentName("prod"); + assertEquals("dev", environment.getEmbeddedEnvironmentName()); + copy.getConfigurationFiles().add("local.json"); + assertTrue(environment.getConfigurationFiles().isEmpty()); + } + + @Test + void modifyVariablesAppliesDefaultsThenFilesThenProjectVariables() throws Exception { + String name = unique("dev"); + String configPath = writeConfigFile(name, "DB_HOST", "db.example", "LOG_LEVEL", "FromFile"); + Project project = projectWithSample(name); + project.setDescribedVariable(new DescribedVariable("LOG_LEVEL", "FromProject", "level")); + register(name, name, List.of(configPath)); + + IVariables variables = new Variables(); + project.modifyVariables(variables, projectConfig(), List.of(configPath), name); + + assertEquals("FromProject", variables.getVariable("LOG_LEVEL")); + assertEquals("FromDefinition", variables.getVariable("OPTIONAL_VAR")); + assertEquals("db.example", variables.getVariable("DB_HOST")); + assertEquals("change-to-your-password", variables.getVariable("DB_PASSWORD")); + } + + @Test + void unlinkedEnvironmentDoesNotApplyEmbeddedDefaults() throws Exception { + String name = unique("plain"); + String configPath = writeConfigFile(name, "FILE_ONLY", "yes"); + Project project = projectWithSample("dev"); + register(name, null, List.of(configPath)); + + IVariables variables = new Variables(); + project.modifyVariables(variables, projectConfig(), List.of(configPath), name); + + assertEquals("yes", variables.getVariable("FILE_ONLY")); + assertNull(variables.getVariable("OPTIONAL_VAR")); + assertNull(variables.getVariable("DB_HOST")); + } + + @Test + void unknownEmbeddedEnvironmentIsLoggedAndSkipped() throws Exception { + String name = unique("missing"); + Project project = projectWithSample("dev"); + register(name, "does-not-exist", List.of()); + + int from = HopLogStore.getLastBufferLineNr(); + IVariables variables = new Variables(); + project.modifyVariables(variables, projectConfig(), List.of(), name); + + assertNull(variables.getVariable("OPTIONAL_VAR")); + assertTrue(logContains(from, "does-not-exist")); + assertTrue(logContains(from, "is not defined in this project")); + } + + @Test + void missingEnvironmentNameSkipsEmbeddedDefaults() throws Exception { + Project project = projectWithSample("dev"); + IVariables variables = new Variables(); + project.modifyVariables(variables, projectConfig(), List.of(), null); + assertNull(variables.getVariable("OPTIONAL_VAR")); + } + + private EmbeddedEnvironment sampleEnvironment(String name) { + EmbeddedEnvironment environment = new EmbeddedEnvironment(); + environment.setName(name); + environment.setDescription("Developer workstation"); + environment + .getVariables() + .add(new EmbeddedEnvironmentVariable("OPTIONAL_VAR", "FromDefinition", "Optional")); + environment + .getVariables() + .add(new EmbeddedEnvironmentVariable("LOG_LEVEL", "FromDefinition", "Hop log level")); + environment + .getMandatoryVariables() + .add(new EmbeddedEnvironmentVariable("DB_HOST", "specify the database host", "JDBC host")); + environment + .getSecretVariables() + .add( + new EmbeddedEnvironmentVariable( + "DB_PASSWORD", "change-to-your-password", "JDBC password")); + return environment; + } + + private Project projectWithSample(String embeddedName) { + Project project = new Project(); + project.getEmbeddedEnvironments().add(sampleEnvironment(embeddedName)); + return project; + } + + private ProjectConfig projectConfig() { + return new ProjectConfig("embedded-project", tempRoot.toString(), "project-config.json"); + } + + private void register(String environmentName, String embeddedName, List files) { + LifecycleEnvironment environment = + new LifecycleEnvironment(environmentName, "", "embedded-project", new ArrayList<>(files)); + environment.setEmbeddedEnvironmentName(embeddedName); + ProjectsConfigSingleton.getConfig().addEnvironment(environment); + registeredEnvironments.add(environmentName); + } + + private String writeConfigFile(String environmentName, String... nameValuePairs) + throws Exception { + String path = + EmbeddedEnvironmentMaterializer.configFilePath(tempRoot.toString(), environmentName); + List variables = new ArrayList<>(); + for (int i = 0; i < nameValuePairs.length; i += 2) { + variables.add(new DescribedVariable(nameValuePairs[i], nameValuePairs[i + 1], "")); + } + DescribedVariablesConfigFile configFile = new DescribedVariablesConfigFile(path); + configFile.setDescribedVariables(variables); + configFile.saveToFile(); + return path; + } + + private String unique(String suffix) { + return "ee4193-" + suffix + "-" + System.nanoTime(); + } + + private static boolean fileExists(String path) throws Exception { + try (FileObject file = HopVfs.getFileObject(path)) { + return file.exists(); + } + } + + private static boolean logContains(int from, String text) { + List events = + HopLogStore.getLogBufferFromTo( + List.of(LogChannel.GENERAL.getLogChannelId()), + true, + from, + HopLogStore.getLastBufferLineNr()); + for (HopLoggingEvent event : events) { + if (String.valueOf(event.getMessage()).contains(text)) { + return true; + } + } + return false; + } +} From f70e16eaaccb154aa8237199c397eaa276e606e5 Mon Sep 17 00:00:00 2001 From: mattcasters Date: Wed, 7 Oct 2026 13:06:07 +0200 Subject: [PATCH 3/4] issue #4193 --- .../pages/projects/projects-environments.adoc | 5 +- .../EmbeddedEnvironmentImporter.java | 205 +++++++++++ .../EmbeddedEnvironmentMaterializer.java | 2 +- .../EnvironmentConfigFileSummary.java | 15 +- .../ImportEmbeddedEnvironmentsDialog.java | 319 ++++++++++++++++++ .../hop/projects/project/ProjectDialog.java | 158 ++++++++- .../messages/messages_en_US.properties | 20 ++ .../EmbeddedEnvironmentImporterTest.java | 181 ++++++++++ 8 files changed, 899 insertions(+), 6 deletions(-) create mode 100644 plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentImporter.java create mode 100644 plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ImportEmbeddedEnvironmentsDialog.java create mode 100644 plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentImporterTest.java diff --git a/docs/hop-user-manual/modules/ROOT/pages/projects/projects-environments.adoc b/docs/hop-user-manual/modules/ROOT/pages/projects/projects-environments.adoc index 9dbd6ad55ea..0d90c92851c 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/projects/projects-environments.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/projects/projects-environments.adoc @@ -127,8 +127,11 @@ Do not put a real password, host, or path in the project file. |Secrets|Secret variables such as passwords and tokens. The default is a placeholder, not the secret. |=== -*Add*, *Edit...*, and *Delete* change the list. +*Add*, *Edit...*, *Delete*, and *Import...* change the list. *Edit...* sets the name, the description, and the three variable lists. +*Import...* reads one or more lifecycle environments of this project from the Hop configuration on this computer. +For each variable in those configuration files you choose *Variable*, *Mandatory*, or *Secret*. +The values in the configuration files are not copied into the project. When a project is added from a folder or from Git and `project-config.json` already defines environments, Hop asks whether to create those lifecycle environments on this computer. A project with no embedded environments still asks the generic question about creating one environment, except when the project is read only or was added from Git. diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentImporter.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentImporter.java new file mode 100644 index 00000000000..ca23e51ef7a --- /dev/null +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentImporter.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.projects.environment; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import lombok.Getter; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.vfs2.FileObject; +import org.apache.hop.core.config.DescribedVariablesConfigFile; +import org.apache.hop.core.variables.DescribedVariable; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.vfs.HopVfs; + +/** + * Builds embedded environment definitions from lifecycle environments already stored in the Hop + * configuration. Variable values are read only to decide whether a variable looks like a secret. + * They are not copied into the project file. + */ +public final class EmbeddedEnvironmentImporter { + + /** Which of the three embedded lists a variable is imported into. */ + public enum Kind { + VARIABLE, + MANDATORY, + SECRET + } + + private EmbeddedEnvironmentImporter() {} + + /** + * Read the configuration files of each environment. A missing or unreadable file is recorded and + * does not stop the others. + * + * @param environments lifecycle environments selected by the user + * @param variables variable space used to resolve configuration file paths + * @return one source per environment that has a name, in the given order + */ + public static List read( + List environments, IVariables variables) { + List sources = new ArrayList<>(); + if (environments == null) { + return sources; + } + for (LifecycleEnvironment environment : environments) { + if (environment == null || StringUtils.isBlank(environment.getName())) { + continue; + } + Map byName = new LinkedHashMap<>(); + List unreadable = new ArrayList<>(); + List files = environment.getConfigurationFiles(); + if (files != null) { + for (String filename : files) { + readFile(filename, variables, byName, unreadable); + } + } + sources.add( + new EnvironmentSource( + environment.getName().trim(), + StringUtils.trimToNull(environment.getPurpose()), + new ArrayList<>(byName.values()), + unreadable)); + } + return sources; + } + + /** + * @param name environment name + * @param description environment description, a placeholder default is not taken from the files + * @param assignments variables and the list each one belongs to + * @return a normalized embedded environment. Default values are left empty. + */ + public static EmbeddedEnvironment toEmbeddedEnvironment( + String name, String description, List assignments) { + EmbeddedEnvironment environment = new EmbeddedEnvironment(); + environment.setName(name); + environment.setDescription(description); + if (assignments != null) { + for (VariableAssignment assignment : assignments) { + if (assignment == null || StringUtils.isBlank(assignment.getName())) { + continue; + } + EmbeddedEnvironmentVariable variable = + new EmbeddedEnvironmentVariable( + assignment.getName(), null, assignment.getDescription()); + Kind kind = assignment.getKind() == null ? Kind.VARIABLE : assignment.getKind(); + switch (kind) { + case MANDATORY -> environment.getMandatoryVariables().add(variable); + case SECRET -> environment.getSecretVariables().add(variable); + default -> environment.getVariables().add(variable); + } + } + } + EmbeddedEnvironmentValidator.normalize(environment); + return environment; + } + + private static void readFile( + String filename, + IVariables variables, + Map byName, + List unreadable) { + if (StringUtils.isBlank(filename)) { + return; + } + String resolved = variables == null ? filename : variables.resolve(filename); + try (FileObject file = HopVfs.getFileObject(resolved)) { + if (!file.exists()) { + unreadable.add(filename); + return; + } + } catch (Exception e) { + unreadable.add(filename); + return; + } + try { + DescribedVariablesConfigFile configFile = new DescribedVariablesConfigFile(resolved); + configFile.readFromFile(); + for (DescribedVariable variable : configFile.getDescribedVariables()) { + addVariable(byName, variable); + } + } catch (Exception e) { + unreadable.add(filename); + } + } + + private static void addVariable( + Map byName, DescribedVariable variable) { + if (variable == null || StringUtils.isBlank(variable.getName())) { + return; + } + String name = variable.getName().trim(); + Kind kind = + EnvironmentConfigFileSummary.looksLikeSecret(name, variable.getValue()) + ? Kind.SECRET + : Kind.VARIABLE; + VariableAssignment existing = byName.get(name); + if (existing == null) { + byName.put( + name, + new VariableAssignment(name, StringUtils.trimToNull(variable.getDescription()), kind)); + return; + } + if (existing.description == null) { + existing.description = StringUtils.trimToNull(variable.getDescription()); + } + if (kind == Kind.SECRET) { + existing.kind = Kind.SECRET; + } + } + + /** One lifecycle environment and the variables found in its configuration files. */ + @Getter + public static final class EnvironmentSource { + private final String name; + private final String description; + private final List variables; + private final List unreadableFiles; + + public EnvironmentSource( + String name, + String description, + List variables, + List unreadableFiles) { + this.name = name; + this.description = description; + this.variables = variables == null ? new ArrayList<>() : variables; + this.unreadableFiles = unreadableFiles == null ? new ArrayList<>() : unreadableFiles; + } + } + + /** + * A variable to place in one of the three lists. {@code kind} is a suggestion until the user + * confirms it. The description may be updated while files are merged. + */ + @Getter + public static final class VariableAssignment { + private final String name; + private String description; + private Kind kind; + + public VariableAssignment(String name, String description, Kind kind) { + this.name = name; + this.description = description; + this.kind = kind == null ? Kind.VARIABLE : kind; + } + } +} diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentMaterializer.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentMaterializer.java index 1864a5a83ba..4cf90e88901 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentMaterializer.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentMaterializer.java @@ -143,7 +143,7 @@ public static List variablesToStore(EmbeddedEnvironment envir /** * Create a lifecycle environment for every embedded definition that is not already registered. * Existing configuration files are left as they are. Nothing is saved to {@code hop-config.json}; - * the caller registers {@link MaterializeResult#getCreated()}. + * the caller registers the environments in the returned {@link MaterializeResult}. * * @param config environments already on this computer * @param project project that holds the definitions diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EnvironmentConfigFileSummary.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EnvironmentConfigFileSummary.java index 99ca327e03e..227171b349f 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EnvironmentConfigFileSummary.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EnvironmentConfigFileSummary.java @@ -216,15 +216,24 @@ private static String formatVariable(DescribedVariable variable) { return line; } - private static boolean maskValue(DescribedVariable variable) { - String name = variable.getName(); + /** + * A variable whose name looks like a secret, or whose value is an encrypted Hop password. + * + * @param name variable name, may be null + * @param value variable value, may be null + * @return true when the value should not be shown or copied into a project file + */ + static boolean looksLikeSecret(String name, String value) { if (name != null && SECRET_NAME.matcher(name).matches()) { return true; } - String value = variable.getValue(); return value != null && value.startsWith(Encr.PASSWORD_ENCRYPTED_PREFIX); } + private static boolean maskValue(DescribedVariable variable) { + return variable != null && looksLikeSecret(variable.getName(), variable.getValue()); + } + private static String message(String key) { return BaseMessages.getString(PKG, key); } diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ImportEmbeddedEnvironmentsDialog.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ImportEmbeddedEnvironmentsDialog.java new file mode 100644 index 00000000000..dd432272941 --- /dev/null +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ImportEmbeddedEnvironmentsDialog.java @@ -0,0 +1,319 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.projects.project; + +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.Const; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.projects.environment.EmbeddedEnvironment; +import org.apache.hop.projects.environment.EmbeddedEnvironmentImporter; +import org.apache.hop.projects.environment.EmbeddedEnvironmentImporter.EnvironmentSource; +import org.apache.hop.projects.environment.EmbeddedEnvironmentImporter.Kind; +import org.apache.hop.projects.environment.EmbeddedEnvironmentImporter.VariableAssignment; +import org.apache.hop.projects.util.Defaults; +import org.apache.hop.ui.core.ConstUi; +import org.apache.hop.ui.core.PropsUi; +import org.apache.hop.ui.core.dialog.BaseDialog; +import org.apache.hop.ui.core.gui.GuiResource; +import org.apache.hop.ui.core.widget.ColumnInfo; +import org.apache.hop.ui.core.widget.TableView; +import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; +import org.apache.hop.ui.util.HelpUtils; +import org.eclipse.swt.SWT; +import org.eclipse.swt.layout.FormAttachment; +import org.eclipse.swt.layout.FormData; +import org.eclipse.swt.layout.FormLayout; +import org.eclipse.swt.widgets.Button; +import org.eclipse.swt.widgets.Control; +import org.eclipse.swt.widgets.Dialog; +import org.eclipse.swt.widgets.Label; +import org.eclipse.swt.widgets.Shell; +import org.eclipse.swt.widgets.TableItem; + +/** + * Asks how each variable from the selected lifecycle environments should be stored: as a variable, + * as mandatory, or as a secret. Values from the configuration files are not shown or copied. + */ +public class ImportEmbeddedEnvironmentsDialog extends Dialog { + private static final Class PKG = ImportEmbeddedEnvironmentsDialog.class; + + private final List sources; + private final IVariables variables; + private final PropsUi props; + + private Shell shell; + private TableView wVariables; + private List returnValue; + + public ImportEmbeddedEnvironmentsDialog( + Shell parent, List sources, IVariables variables) { + super(parent, SWT.DIALOG_TRIM | SWT.APPLICATION_MODAL | SWT.RESIZE); + this.sources = sources == null ? List.of() : sources; + this.variables = variables; + this.props = PropsUi.getInstance(); + } + + /** + * @return the embedded environments to add, or null when cancelled + */ + public List open() { + Shell parent = getParent(); + shell = new Shell(parent, SWT.DIALOG_TRIM | SWT.APPLICATION_MODAL | SWT.RESIZE); + shell.setImage( + GuiResource.getInstance() + .getImage( + "environment.svg", + PKG.getClassLoader(), + ConstUi.SMALL_ICON_SIZE, + ConstUi.SMALL_ICON_SIZE)); + PropsUi.setLook(shell); + + int margin = PropsUi.getMargin() + 2; + FormLayout formLayout = new FormLayout(); + formLayout.marginWidth = PropsUi.getFormMargin(); + formLayout.marginHeight = PropsUi.getFormMargin(); + shell.setLayout(formLayout); + shell.setText(BaseMessages.getString(PKG, "ImportEmbeddedEnvironmentsDialog.Shell.Name")); + + Button wOk = new Button(shell, SWT.PUSH); + wOk.setText(BaseMessages.getString(PKG, "System.Button.OK")); + wOk.addListener(SWT.Selection, event -> ok()); + Button wCancel = new Button(shell, SWT.PUSH); + wCancel.setText(BaseMessages.getString(PKG, "System.Button.Cancel")); + wCancel.addListener(SWT.Selection, event -> cancel()); + BaseTransformDialog.positionBottomButtons(shell, new Button[] {wOk, wCancel}, margin * 3, null); + HelpUtils.createHelpButton(shell, Const.getDocUrl(Defaults.DOCUMENTATION_URI)); + + Label explanation = new Label(shell, SWT.LEFT | SWT.WRAP); + PropsUi.setLook(explanation); + explanation.setText( + BaseMessages.getString(PKG, "ImportEmbeddedEnvironmentsDialog.Explanation")); + FormData fdExplanation = new FormData(); + fdExplanation.left = new FormAttachment(0, 0); + fdExplanation.right = new FormAttachment(100, 0); + fdExplanation.top = new FormAttachment(0, 0); + explanation.setLayoutData(fdExplanation); + + Control last = explanation; + String unreadable = unreadableFiles(); + if (unreadable != null) { + Label warning = new Label(shell, SWT.LEFT | SWT.WRAP); + PropsUi.setLook(warning); + warning.setText(unreadable); + FormData fdWarning = new FormData(); + fdWarning.left = new FormAttachment(0, 0); + fdWarning.right = new FormAttachment(100, 0); + fdWarning.top = new FormAttachment(explanation, margin); + warning.setLayoutData(fdWarning); + last = warning; + } + + List rows = rowsInOrder(); + wVariables = + new TableView( + variables, + shell, + SWT.BORDER | SWT.FULL_SELECTION | SWT.SINGLE, + columns(), + Math.max(rows.size(), 1), + false, + null, + props, + true, + null, + false, + false); + PropsUi.setLook(wVariables); + FormData fdVariables = new FormData(); + fdVariables.left = new FormAttachment(0, 0); + fdVariables.right = new FormAttachment(100, 0); + fdVariables.top = new FormAttachment(last, margin); + fdVariables.bottom = new FormAttachment(wOk, -margin * 2); + wVariables.setLayoutData(fdVariables); + fillRows(rows); + + shell.setMinimumSize(760, 480); + shell.setDefaultButton(wOk); + BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel()); + return returnValue; + } + + private void ok() { + List edited = readRows(); + List environments = new ArrayList<>(); + for (EnvironmentSource source : sources) { + List forSource = new ArrayList<>(); + for (Row row : edited) { + if (source.getName().equals(row.environmentName)) { + forSource.add(new VariableAssignment(row.name, row.description, row.kind)); + } + } + environments.add( + EmbeddedEnvironmentImporter.toEmbeddedEnvironment( + source.getName(), source.getDescription(), forSource)); + } + returnValue = environments; + dispose(); + } + + private void cancel() { + returnValue = null; + dispose(); + } + + private void dispose() { + shell.dispose(); + } + + private ColumnInfo[] columns() { + String[] kinds = + new String[] {kindLabel(Kind.VARIABLE), kindLabel(Kind.MANDATORY), kindLabel(Kind.SECRET)}; + ColumnInfo environment = + new ColumnInfo( + BaseMessages.getString(PKG, "ImportEmbeddedEnvironmentsDialog.Column.Environment"), + ColumnInfo.COLUMN_TYPE_TEXT, + false, + true); + ColumnInfo name = + new ColumnInfo( + BaseMessages.getString(PKG, "ImportEmbeddedEnvironmentsDialog.Column.Name"), + ColumnInfo.COLUMN_TYPE_TEXT, + false, + true); + ColumnInfo description = + new ColumnInfo( + BaseMessages.getString(PKG, "ImportEmbeddedEnvironmentsDialog.Column.Description"), + ColumnInfo.COLUMN_TYPE_TEXT, + false, + false); + ColumnInfo kind = + new ColumnInfo( + BaseMessages.getString(PKG, "ImportEmbeddedEnvironmentsDialog.Column.Kind"), + ColumnInfo.COLUMN_TYPE_CCOMBO, + kinds, + true); + return new ColumnInfo[] {environment, name, description, kind}; + } + + private void fillRows(List rows) { + for (int i = 0; i < rows.size() && i < wVariables.table.getItemCount(); i++) { + Row row = rows.get(i); + TableItem item = wVariables.table.getItem(i); + item.setText(1, Const.NVL(row.environmentName, "")); + item.setText(2, Const.NVL(row.name, "")); + item.setText(3, Const.NVL(row.description, "")); + item.setText(4, kindLabel(row.kind)); + } + wVariables.setRowNums(); + wVariables.optWidth(true); + } + + private List readRows() { + List rows = new ArrayList<>(); + for (int i = 0; i < wVariables.nrNonEmpty(); i++) { + TableItem item = wVariables.getNonEmpty(i); + if (StringUtils.isBlank(item.getText(2))) { + continue; + } + rows.add( + new Row( + item.getText(1), + item.getText(2), + StringUtils.trimToNull(item.getText(3)), + kindOf(item.getText(4)))); + } + return rows; + } + + private List rowsInOrder() { + List rows = new ArrayList<>(); + for (EnvironmentSource source : sources) { + for (VariableAssignment assignment : source.getVariables()) { + rows.add( + new Row( + source.getName(), + assignment.getName(), + assignment.getDescription(), + assignment.getKind())); + } + } + return rows; + } + + /** One table row: the environment it belongs to and the list the user chose. */ + private static final class Row { + private final String environmentName; + private final String name; + private final String description; + private final Kind kind; + + private Row(String environmentName, String name, String description, Kind kind) { + this.environmentName = environmentName; + this.name = name; + this.description = description; + this.kind = kind; + } + } + + private String unreadableFiles() { + List files = new ArrayList<>(); + for (EnvironmentSource source : sources) { + for (String filename : source.getUnreadableFiles()) { + if (StringUtils.isNotBlank(filename) && !files.contains(filename)) { + files.add(filename); + } + } + } + if (files.isEmpty()) { + return null; + } + int shown = Math.min(files.size(), 8); + StringBuilder text = new StringBuilder(); + text.append(BaseMessages.getString(PKG, "ImportEmbeddedEnvironmentsDialog.Unreadable")); + for (int i = 0; i < shown; i++) { + text.append(Const.CR).append(files.get(i)); + } + if (files.size() > shown) { + text.append(Const.CR).append("..."); + } + return text.toString(); + } + + private static String kindLabel(Kind kind) { + Kind resolved = kind == null ? Kind.VARIABLE : kind; + return switch (resolved) { + case MANDATORY -> + BaseMessages.getString(PKG, "ImportEmbeddedEnvironmentsDialog.Kind.Mandatory"); + case SECRET -> BaseMessages.getString(PKG, "ImportEmbeddedEnvironmentsDialog.Kind.Secret"); + default -> BaseMessages.getString(PKG, "ImportEmbeddedEnvironmentsDialog.Kind.Variable"); + }; + } + + private static Kind kindOf(String label) { + if (kindLabel(Kind.MANDATORY).equals(label)) { + return Kind.MANDATORY; + } + if (kindLabel(Kind.SECRET).equals(label)) { + return Kind.SECRET; + } + return Kind.VARIABLE; + } +} diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ProjectDialog.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ProjectDialog.java index aafdb33892c..44dc65cd93b 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ProjectDialog.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ProjectDialog.java @@ -37,7 +37,10 @@ import org.apache.hop.projects.config.ProjectsConfig; import org.apache.hop.projects.config.ProjectsConfigSingleton; import org.apache.hop.projects.environment.EmbeddedEnvironment; +import org.apache.hop.projects.environment.EmbeddedEnvironmentImporter; +import org.apache.hop.projects.environment.EmbeddedEnvironmentImporter.EnvironmentSource; import org.apache.hop.projects.environment.EmbeddedEnvironmentValidator; +import org.apache.hop.projects.environment.LifecycleEnvironment; import org.apache.hop.projects.gui.ProjectsGuiPlugin; import org.apache.hop.projects.util.Defaults; import org.apache.hop.projects.util.ProjectRenameBlockedException; @@ -45,6 +48,7 @@ import org.apache.hop.ui.core.ConstUi; import org.apache.hop.ui.core.PropsUi; import org.apache.hop.ui.core.dialog.BaseDialog; +import org.apache.hop.ui.core.dialog.EnterSelectionDialog; import org.apache.hop.ui.core.dialog.ErrorDialog; import org.apache.hop.ui.core.dialog.MessageBox; import org.apache.hop.ui.core.gui.GuiResource; @@ -111,6 +115,7 @@ public class ProjectDialog extends Dialog { private Button wAddEnvironment; private Button wEditEnvironment; private Button wDeleteEnvironment; + private Button wImportEnvironment; /** Working copy of the embedded environments. Applied to the project on OK. */ private final List embeddedEnvironments = new ArrayList<>(); @@ -719,8 +724,17 @@ private void createEnvironmentsTab(CTabFolder folder, int margin) { wDeleteEnvironment.setText( BaseMessages.getString(PKG, "ProjectDialog.Button.DeleteEnvironment")); wDeleteEnvironment.addListener(SWT.Selection, event -> deleteEmbeddedEnvironment()); + wImportEnvironment = new Button(comp, SWT.PUSH); + wImportEnvironment.setText( + BaseMessages.getString(PKG, "ProjectDialog.Button.ImportEnvironment")); + wImportEnvironment.setToolTipText( + BaseMessages.getString(PKG, "ProjectDialog.Button.ImportEnvironment.Tooltip")); + wImportEnvironment.addListener(SWT.Selection, event -> importEmbeddedEnvironments()); BaseTransformDialog.positionBottomButtons( - comp, new Button[] {wAddEnvironment, wEditEnvironment, wDeleteEnvironment}, margin, null); + comp, + new Button[] {wAddEnvironment, wEditEnvironment, wDeleteEnvironment, wImportEnvironment}, + margin, + null); ColumnInfo[] columnInfo = new ColumnInfo[] { @@ -813,6 +827,147 @@ private void deleteEmbeddedEnvironment() { needingProjectRefresh = true; } + private void importEmbeddedEnvironments() { + if (!environmentsEditable()) { + return; + } + try { + List available = lifecycleEnvironmentsOfThisProject(); + if (available.isEmpty()) { + MessageBox box = new MessageBox(shell, SWT.OK | SWT.ICON_INFORMATION); + box.setText(BaseMessages.getString(PKG, "ProjectDialog.Environments.Import.None.Header")); + box.setMessage( + BaseMessages.getString(PKG, "ProjectDialog.Environments.Import.None.Message")); + box.open(); + return; + } + List chosen = chooseLifecycleEnvironments(available); + if (chosen.isEmpty()) { + return; + } + List sources = EmbeddedEnvironmentImporter.read(chosen, variables); + ImportEmbeddedEnvironmentsDialog dialog = + new ImportEmbeddedEnvironmentsDialog(shell, sources, variables); + List imported = dialog.open(); + if (imported == null) { + return; + } + applyImportedEnvironments(imported); + } catch (Exception e) { + new ErrorDialog( + shell, + BaseMessages.getString(PKG, "ProjectDialog.Environments.Import.Error.Header"), + BaseMessages.getString(PKG, "ProjectDialog.Environments.Import.Error.Message"), + e); + } + } + + private List chooseLifecycleEnvironments( + List available) { + String[] choices = new String[available.size()]; + for (int i = 0; i < available.size(); i++) { + choices[i] = Const.NVL(available.get(i).getName(), ""); + } + EnterSelectionDialog selectionDialog = + new EnterSelectionDialog( + shell, + choices, + BaseMessages.getString(PKG, "ProjectDialog.Environments.Import.Select.Header"), + BaseMessages.getString(PKG, "ProjectDialog.Environments.Import.Select.Message")); + selectionDialog.setMulti(true); + if (selectionDialog.open() == null) { + return List.of(); + } + int[] indices = selectionDialog.getSelectionIndeces(); + List chosen = new ArrayList<>(); + if (indices == null) { + return chosen; + } + for (int index : indices) { + if (index >= 0 && index < available.size()) { + chosen.add(available.get(index)); + } + } + return chosen; + } + + /** + * Lifecycle environments registered for the project this dialog is editing. A name typed in the + * dialog and not yet saved does not change which project the environments belong to. + */ + private List lifecycleEnvironmentsOfThisProject() { + String projectName = StringUtils.trimToNull(projectConfig.getProjectName()); + if (projectName == null) { + return List.of(); + } + List found = + ProjectsConfigSingleton.getConfig().findEnvironmentsOfProject(projectName); + if (found == null || found.isEmpty()) { + return List.of(); + } + List named = new ArrayList<>(); + for (LifecycleEnvironment environment : found) { + if (environment != null && StringUtils.isNotBlank(environment.getName())) { + named.add(environment); + } + } + return named; + } + + private void applyImportedEnvironments(List imported) { + List existing = new ArrayList<>(); + for (EmbeddedEnvironment environment : imported) { + if (environment != null && indexOfEmbeddedEnvironment(environment.getName()) >= 0) { + existing.add(environment.getName()); + } + } + boolean replace = true; + if (!existing.isEmpty()) { + MessageBox box = new MessageBox(shell, SWT.YES | SWT.NO | SWT.ICON_QUESTION); + box.setText(BaseMessages.getString(PKG, "ProjectDialog.Environments.Import.Replace.Header")); + box.setMessage( + BaseMessages.getString( + PKG, + "ProjectDialog.Environments.Import.Replace.Message", + String.join(Const.CR, existing))); + replace = (box.open() & SWT.YES) != 0; + } + int lastIndex = -1; + for (EmbeddedEnvironment environment : imported) { + if (environment == null || StringUtils.isBlank(environment.getName())) { + continue; + } + int index = indexOfEmbeddedEnvironment(environment.getName()); + if (index >= 0) { + if (!replace) { + continue; + } + embeddedEnvironments.set(index, environment); + lastIndex = index; + } else { + embeddedEnvironments.add(environment); + lastIndex = embeddedEnvironments.size() - 1; + } + } + if (lastIndex >= 0) { + refreshEmbeddedEnvironments(lastIndex); + needingProjectRefresh = true; + } + } + + private int indexOfEmbeddedEnvironment(String name) { + if (StringUtils.isBlank(name)) { + return -1; + } + String trimmed = name.trim(); + for (int i = 0; i < embeddedEnvironments.size(); i++) { + if (trimmed.equals(StringUtils.trimToEmpty(embeddedEnvironments.get(i).getName()))) { + return i; + } + } + return -1; + } + private boolean environmentsEditable() { return wAddEnvironment != null && wAddEnvironment.isEnabled(); } @@ -888,6 +1043,7 @@ private void updateReadOnlyWidgets() { wAddEnvironment.setEnabled(editable); wEditEnvironment.setEnabled(editable); wDeleteEnvironment.setEnabled(editable); + wImportEnvironment.setEnabled(editable); } updateAutoExportMetadataWidgets(); updateParentFolderWidgets(); diff --git a/plugins/misc/projects/src/main/resources/org/apache/hop/projects/project/messages/messages_en_US.properties b/plugins/misc/projects/src/main/resources/org/apache/hop/projects/project/messages/messages_en_US.properties index 34fa382e5cf..170765c7d92 100644 --- a/plugins/misc/projects/src/main/resources/org/apache/hop/projects/project/messages/messages_en_US.properties +++ b/plugins/misc/projects/src/main/resources/org/apache/hop/projects/project/messages/messages_en_US.properties @@ -94,6 +94,26 @@ ProjectDialog.Environments.Column.Description=Description ProjectDialog.Button.AddEnvironment=Add ProjectDialog.Button.EditEnvironment=Edit... ProjectDialog.Button.DeleteEnvironment=Delete +ProjectDialog.Button.ImportEnvironment=Import... +ProjectDialog.Button.ImportEnvironment.Tooltip=Import one or more lifecycle environments of this project +ProjectDialog.Environments.Import.None.Header=No environments +ProjectDialog.Environments.Import.None.Message=This project has no lifecycle environments in the Hop configuration. +ProjectDialog.Environments.Import.Select.Header=Import environments +ProjectDialog.Environments.Import.Select.Message=Select the lifecycle environments of this project to import. Their configuration files supply the variable names. +ProjectDialog.Environments.Import.Replace.Header=Replace environment definitions? +ProjectDialog.Environments.Import.Replace.Message=These environment definitions already exist in the project:\n{0}\n\nReplace them? Definitions that are not in this list are still imported. +ProjectDialog.Environments.Import.Error.Header=Error +ProjectDialog.Environments.Import.Error.Message=The environments could not be imported. +ImportEmbeddedEnvironmentsDialog.Shell.Name=Import environment definitions +ImportEmbeddedEnvironmentsDialog.Explanation=Values from the configuration files are not copied. Mark each variable as a variable, mandatory, or a secret. You can set placeholder defaults afterwards. +ImportEmbeddedEnvironmentsDialog.Unreadable=These configuration files could not be read. Their variables are not listed. +ImportEmbeddedEnvironmentsDialog.Column.Environment=Environment +ImportEmbeddedEnvironmentsDialog.Column.Name=Name +ImportEmbeddedEnvironmentsDialog.Column.Description=Description +ImportEmbeddedEnvironmentsDialog.Column.Kind=Use as +ImportEmbeddedEnvironmentsDialog.Kind.Variable=Variable +ImportEmbeddedEnvironmentsDialog.Kind.Mandatory=Mandatory +ImportEmbeddedEnvironmentsDialog.Kind.Secret=Secret ProjectDialog.Environments.Delete.Header=Delete environment? ProjectDialog.Environments.Delete.Message=Delete the environment definition ''{0}'' from this project? ProjectDialog.Environments.MissingName.Header=Environment name diff --git a/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentImporterTest.java b/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentImporterTest.java new file mode 100644 index 00000000000..aeb18f791d3 --- /dev/null +++ b/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentImporterTest.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.projects.environment; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.stream.Stream; +import org.apache.hop.core.config.DescribedVariablesConfigFile; +import org.apache.hop.core.encryption.Encr; +import org.apache.hop.core.variables.DescribedVariable; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.variables.Variables; +import org.apache.hop.projects.environment.EmbeddedEnvironmentImporter.EnvironmentSource; +import org.apache.hop.projects.environment.EmbeddedEnvironmentImporter.Kind; +import org.apache.hop.projects.environment.EmbeddedEnvironmentImporter.VariableAssignment; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class EmbeddedEnvironmentImporterTest { + + private Path tempRoot; + + @BeforeEach + void setUp() throws Exception { + tempRoot = Files.createTempDirectory("hop-import-env"); + } + + @AfterEach + void tearDown() throws Exception { + if (tempRoot != null && Files.exists(tempRoot)) { + try (Stream walk = Files.walk(tempRoot)) { + walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } + tempRoot = null; + } + } + + @Test + void readClassifiesSecretsAndDropsValues() throws Exception { + String configPath = writeConfig("dev.json", variables()); + LifecycleEnvironment environment = + new LifecycleEnvironment("dev", "Developer workstation", "warehouse", List.of(configPath)); + + List environments = new ArrayList<>(); + environments.add(environment); + environments.add(null); + environments.add(new LifecycleEnvironment()); + List sources = + EmbeddedEnvironmentImporter.read(environments, new Variables()); + + assertEquals(1, sources.size()); + EnvironmentSource source = sources.get(0); + assertEquals("dev", source.getName()); + assertEquals("Developer workstation", source.getDescription()); + assertTrue(source.getUnreadableFiles().isEmpty()); + assertEquals(List.of("LOG_LEVEL", "DB_HOST", "DB_PASSWORD", "DB_TOKEN"), names(source)); + assertEquals(Kind.VARIABLE, kind(source, "LOG_LEVEL")); + assertEquals(Kind.VARIABLE, kind(source, "DB_HOST")); + assertEquals(Kind.SECRET, kind(source, "DB_PASSWORD")); + assertEquals(Kind.SECRET, kind(source, "DB_TOKEN")); + assertEquals("Hop log level", description(source, "LOG_LEVEL")); + + EmbeddedEnvironment embedded = + EmbeddedEnvironmentImporter.toEmbeddedEnvironment( + source.getName(), source.getDescription(), source.getVariables()); + assertNull(embedded.getVariables().get(0).getDefaultValue()); + assertEquals("LOG_LEVEL", embedded.getVariables().get(0).getName()); + assertEquals("DB_HOST", embedded.getVariables().get(1).getName()); + assertEquals("DB_PASSWORD", embedded.getSecretVariables().get(0).getName()); + assertNull(embedded.getSecretVariables().get(0).getDefaultValue()); + assertEquals("DB_TOKEN", embedded.getSecretVariables().get(1).getName()); + assertTrue(embedded.getMandatoryVariables().isEmpty()); + } + + @Test + void readMergesFilesAndKeepsAnUnreadablePath() throws Exception { + IVariables variables = new Variables(); + variables.setVariable("ENV_DIR", tempRoot.toString()); + String first = + writeConfig("first.json", List.of(new DescribedVariable("DB_HOST", "db.example", ""))); + String second = + writeConfig( + "second.json", + List.of( + new DescribedVariable("DB_HOST", "other", "JDBC host"), + new DescribedVariable( + "API_KEY", Encr.PASSWORD_ENCRYPTED_PREFIX + "abc", "API key"))); + LifecycleEnvironment environment = + new LifecycleEnvironment( + "prod", " ", "warehouse", List.of("${ENV_DIR}/missing.json", first, second)); + + EnvironmentSource source = + EmbeddedEnvironmentImporter.read(List.of(environment), variables).get(0); + + assertNull(source.getDescription()); + assertEquals(List.of("${ENV_DIR}/missing.json"), source.getUnreadableFiles()); + assertEquals(Kind.VARIABLE, kind(source, "DB_HOST")); + assertEquals("JDBC host", description(source, "DB_HOST")); + assertEquals(Kind.SECRET, kind(source, "API_KEY")); + + EmbeddedEnvironment embedded = + EmbeddedEnvironmentImporter.toEmbeddedEnvironment( + source.getName(), + source.getDescription(), + List.of( + new VariableAssignment("DB_HOST", "JDBC host", Kind.MANDATORY), + new VariableAssignment("API_KEY", "API key", Kind.SECRET), + new VariableAssignment(" ", "ignored", Kind.VARIABLE))); + assertEquals(1, embedded.getMandatoryVariables().size()); + assertEquals("DB_HOST", embedded.getMandatoryVariables().get(0).getName()); + assertNull(embedded.getMandatoryVariables().get(0).getDefaultValue()); + assertEquals("API_KEY", embedded.getSecretVariables().get(0).getName()); + assertTrue(embedded.getVariables().isEmpty()); + } + + private List variables() { + return List.of( + new DescribedVariable("LOG_LEVEL", "Basic", "Hop log level"), + new DescribedVariable("DB_HOST", "db.example", "JDBC host"), + new DescribedVariable("DB_PASSWORD", "real-password", "JDBC password"), + new DescribedVariable("DB_TOKEN", Encr.PASSWORD_ENCRYPTED_PREFIX + "abc", "token"), + new DescribedVariable(" ", "ignored", "ignored")); + } + + private String writeConfig(String filename, List variables) throws Exception { + String path = tempRoot.resolve(filename).toString(); + DescribedVariablesConfigFile configFile = new DescribedVariablesConfigFile(path); + configFile.setDescribedVariables(new ArrayList<>(variables)); + configFile.saveToFile(); + return path; + } + + private static List names(EnvironmentSource source) { + List names = new ArrayList<>(); + for (VariableAssignment assignment : source.getVariables()) { + names.add(assignment.getName()); + } + return names; + } + + private static Kind kind(EnvironmentSource source, String name) { + return assignment(source, name).getKind(); + } + + private static String description(EnvironmentSource source, String name) { + return assignment(source, name).getDescription(); + } + + private static VariableAssignment assignment(EnvironmentSource source, String name) { + for (VariableAssignment assignment : source.getVariables()) { + if (name.equals(assignment.getName())) { + return assignment; + } + } + throw new AssertionError("Missing variable " + name); + } +} From d2e490c37db1d3ab5999d918e5393f9b9d6c29e1 Mon Sep 17 00:00:00 2001 From: mattcasters Date: Wed, 7 Oct 2026 16:24:36 +0200 Subject: [PATCH 4/4] Issue #4193 : Resolve the environment folder before writing configuration files The directory dialog returns a folder such as ${HOP_CONFIG_FOLDER}/environments//. Resolve that folder before building the configuration filename so the file is not written under the Hop installation directory, and keep the variable in the lifecycle environment reference. The Environments tab can implement those environments on this computer and copy a definition. Mandatory and secret variables come from the single variable list. Parent-folder copy and project load or restore log a caught error on one line. --- .../pages/projects/projects-environments.adoc | 50 ++++--- .../environment/EmbeddedEnvironment.java | 91 +++++++++-- .../EmbeddedEnvironmentImporter.java | 77 ++++++---- .../EmbeddedEnvironmentMaterializer.java | 126 ++++++++++++++-- .../EmbeddedEnvironmentValidator.java | 24 +-- .../EmbeddedEnvironmentVariable.java | 19 ++- .../hop/projects/gui/ProjectsGuiPlugin.java | 5 +- .../project/EmbeddedEnvironmentDialog.java | 105 +++++-------- .../ImportEmbeddedEnvironmentsDialog.java | 90 ++++++----- .../ParentProjectFolderSynchronizer.java | 15 +- .../apache/hop/projects/project/Project.java | 8 +- .../hop/projects/project/ProjectDialog.java | 141 +++++++++++++++++- .../hop/projects/util/ProjectsUtil.java | 26 +++- .../gui/messages/messages_en_US.properties | 2 +- .../messages/messages_en_US.properties | 42 ++++-- .../EmbeddedEnvironmentImporterTest.java | 78 +++++++--- .../EmbeddedEnvironmentMaterializerTest.java | 120 +++++++++++++-- .../environment/EmbeddedEnvironmentTest.java | 48 +++--- .../ParentProjectFolderSynchronizerTest.java | 29 ++++ .../project/ProjectReferencesTest.java | 30 ++++ .../hop/projects/project/ProjectTest.java | 86 +++++++++-- 21 files changed, 903 insertions(+), 309 deletions(-) diff --git a/docs/hop-user-manual/modules/ROOT/pages/projects/projects-environments.adoc b/docs/hop-user-manual/modules/ROOT/pages/projects/projects-environments.adoc index 0d90c92851c..9500202e86b 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/projects/projects-environments.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/projects/projects-environments.adoc @@ -111,36 +111,42 @@ image:hop-gui/environment/create-project-dialog-variables-tab.png[Project Proper The *Environments* tab stores zero, one, or more lifecycle environment definitions in `project-config.json`. These definitions are meant to be checked in with the project. -Each one has a name, a description, and three lists of variables. -A variable name belongs to only one of the three lists. +Each one has a name, a description, and a list of variables. +Each variable has a *Mandatory* flag and a *Secret* flag. +A mandatory variable is one a person has to set on their own computer. +A secret is a password, token, or similar value. +Both flags can be set on the same variable. The definition does not contain configuration files. -The *Default value* of a variable is a placeholder, for example `change-to-your-password` or `specify the folder on the system where the files are`. -Do not put a real password, host, or path in the project file. +The *Default value* of a variable is checked in with the project. +A value that is the same on every computer, such as a log level or a host name, can be stored as it is. +Use a placeholder such as `change-to-your-password` when the real value should stay on one computer. +Clear a real secret before you save when it should stay on this computer. -.Embedded environment variable lists -[cols="20%,80%",options="header"] -|=== -|List|What it is for -|Variables|Optional variables. The default is documentation for people who use the project. -|Mandatory|Variables a person has to fill in on their own computer. The default is still only a placeholder. -|Secrets|Secret variables such as passwords and tokens. The default is a placeholder, not the secret. -|=== - -*Add*, *Edit...*, *Delete*, and *Import...* change the list. -*Edit...* sets the name, the description, and the three variable lists. +*Add*, *Edit...*, *Copy...*, *Delete*, and *Import...* change the list. +*Edit...* sets the name, the description, and the variables. +*Copy...* asks for a new name and copies the description and the variables. *Import...* reads one or more lifecycle environments of this project from the Hop configuration on this computer. -For each variable in those configuration files you choose *Variable*, *Mandatory*, or *Secret*. -The values in the configuration files are not copied into the project. +For each variable in those configuration files you set *Secret* and *Mandatory*. +The value from the configuration file is copied in as the *Default value*, including a secret. +You can change or clear that default before it is saved in the project. +When the same name appears in more than one configuration file, the last value that is not blank is kept. +A blank value does not clear an earlier one. +The first description that is not blank is kept. + +*Implement* creates the lifecycle environments on this computer from the definitions in the list. +It asks the same question as when a project is added. +The project file is saved when you click OK. When a project is added from a folder or from Git and `project-config.json` already defines environments, Hop asks whether to create those lifecycle environments on this computer. A project with no embedded environments still asks the generic question about creating one environment, except when the project is read only or was added from Git. -Only the *Mandatory* and *Secrets* variables are written, one configuration file per environment. -Optional variables are applied when the environment is enabled and are not stored in that file. -The value written at creation is the placeholder from the project. -Replace it with the value for this computer. -A value in the configuration file replaces the placeholder. +Every variable marked *Secret* or *Mandatory* is written, one configuration file per environment. +A variable that is neither is applied when the environment is enabled and is not stored in that file. +A secret is written whether or not it is mandatory. +The value written for each of those variables is its default from the project, including a placeholder such as `change-to-your-password`. +Replace it on this computer when that default is only a placeholder. +A value in the configuration file replaces the default. A variable with the same name on the project still wins. The files are written to the folder in the project variable `+${ENVIRONMENTS_FOLDER}+` when that variable is set. diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironment.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironment.java index b05010d82e5..5ff157756d6 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironment.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironment.java @@ -17,8 +17,11 @@ package org.apache.hop.projects.environment; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.List; +import lombok.AccessLevel; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -26,8 +29,8 @@ /** * Lifecycle environment definition stored in {@code project-config.json}. * - *

Holds the name, description, and placeholder variable defaults that are shared through version - * control. Configuration files and real values stay on the computer that runs Hop. + *

Holds the name, description, and variable defaults that are shared through version control. + * Configuration files stay on the computer that runs Hop. */ @Getter @Setter @@ -38,28 +41,92 @@ public class EmbeddedEnvironment { private String description; - /** Optional variables. Applied as defaults. Not written to a local configuration file. */ + /** + * Variables for this environment. A variable with {@code mandatory} or {@code secret} set is + * written to the local configuration file. The others are applied as defaults only. + */ private List variables = new ArrayList<>(); - /** Values a person must fill in. Written to the local configuration file when one is created. */ - private List mandatoryVariables = new ArrayList<>(); + /** + * {@code mandatoryVariables} from project files written before mandatory was a flag on each + * variable. Folded into {@code variables} by {@link #absorbLegacyVariables()}. + */ + @JsonIgnore + @Getter(AccessLevel.NONE) + @Setter(AccessLevel.NONE) + private List legacyMandatoryVariables; - /** Secrets. Written to the local configuration file when one is created. Placeholders only. */ - private List secretVariables = new ArrayList<>(); + /** + * {@code secretVariables} from project files written before secret was a flag on each variable. + * Folded into {@code variables} by {@link #absorbLegacyVariables()}. + */ + @JsonIgnore + @Getter(AccessLevel.NONE) + @Setter(AccessLevel.NONE) + private List legacySecretVariables; /** * @return a deep copy */ public EmbeddedEnvironment copy() { + absorbLegacyVariables(); EmbeddedEnvironment copy = new EmbeddedEnvironment(); copy.name = name; copy.description = description; copy.variables = copyVariables(variables); - copy.mandatoryVariables = copyVariables(mandatoryVariables); - copy.secretVariables = copyVariables(secretVariables); return copy; } + /** + * Accept a project file that still stores mandatory variables in their own list. Each of those + * variables is appended to {@code variables} with {@code mandatory} set. + */ + @JsonProperty(value = "mandatoryVariables", access = JsonProperty.Access.WRITE_ONLY) + public void setMandatoryVariables(List legacyMandatoryVariables) { + this.legacyMandatoryVariables = legacyMandatoryVariables; + } + + /** + * Accept a project file that still stores secrets in their own list. Each of those variables is + * appended to {@code variables} with {@code secret} set. + */ + @JsonProperty(value = "secretVariables", access = JsonProperty.Access.WRITE_ONLY) + public void setSecretVariables(List legacySecretVariables) { + this.legacySecretVariables = legacySecretVariables; + } + + /** + * Move {@code mandatoryVariables} and {@code secretVariables} into {@code variables}. Safe to + * call more than once. Mandatory entries are appended first. + */ + public void absorbLegacyVariables() { + absorb(legacyMandatoryVariables, false, true); + legacyMandatoryVariables = null; + absorb(legacySecretVariables, true, false); + legacySecretVariables = null; + } + + private void absorb(List legacy, boolean secret, boolean mandatory) { + if (legacy == null || legacy.isEmpty()) { + return; + } + if (variables == null) { + variables = new ArrayList<>(); + } + for (EmbeddedEnvironmentVariable variable : legacy) { + if (variable == null) { + continue; + } + if (secret) { + variable.setSecret(true); + } + if (mandatory) { + variable.setMandatory(true); + } + variables.add(variable); + } + } + private static List copyVariables( List source) { List copy = new ArrayList<>(); @@ -72,7 +139,11 @@ private static List copyVariables( } copy.add( new EmbeddedEnvironmentVariable( - variable.getName(), variable.getDefaultValue(), variable.getDescription())); + variable.getName(), + variable.getDefaultValue(), + variable.getDescription(), + variable.isMandatory(), + variable.isSecret())); } return copy; } diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentImporter.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentImporter.java index ca23e51ef7a..9232a5ec3dd 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentImporter.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentImporter.java @@ -31,18 +31,11 @@ /** * Builds embedded environment definitions from lifecycle environments already stored in the Hop - * configuration. Variable values are read only to decide whether a variable looks like a secret. - * They are not copied into the project file. + * configuration. The value of each variable is copied in as its default. A later configuration file + * replaces an earlier value for the same name. */ public final class EmbeddedEnvironmentImporter { - /** Which of the three embedded lists a variable is imported into. */ - public enum Kind { - VARIABLE, - MANDATORY, - SECRET - } - private EmbeddedEnvironmentImporter() {} /** @@ -83,9 +76,10 @@ public static List read( /** * @param name environment name - * @param description environment description, a placeholder default is not taken from the files - * @param assignments variables and the list each one belongs to - * @return a normalized embedded environment. Default values are left empty. + * @param description environment description + * @param assignments variables, including whether each one is a secret, and the default copied + * from the configuration file + * @return a normalized embedded environment */ public static EmbeddedEnvironment toEmbeddedEnvironment( String name, String description, List assignments) { @@ -97,15 +91,15 @@ public static EmbeddedEnvironment toEmbeddedEnvironment( if (assignment == null || StringUtils.isBlank(assignment.getName())) { continue; } - EmbeddedEnvironmentVariable variable = - new EmbeddedEnvironmentVariable( - assignment.getName(), null, assignment.getDescription()); - Kind kind = assignment.getKind() == null ? Kind.VARIABLE : assignment.getKind(); - switch (kind) { - case MANDATORY -> environment.getMandatoryVariables().add(variable); - case SECRET -> environment.getSecretVariables().add(variable); - default -> environment.getVariables().add(variable); - } + environment + .getVariables() + .add( + new EmbeddedEnvironmentVariable( + assignment.getName(), + assignment.getDefaultValue(), + assignment.getDescription(), + assignment.isMandatory(), + assignment.isSecret())); } } EmbeddedEnvironmentValidator.normalize(environment); @@ -147,22 +141,25 @@ private static void addVariable( return; } String name = variable.getName().trim(); - Kind kind = - EnvironmentConfigFileSummary.looksLikeSecret(name, variable.getValue()) - ? Kind.SECRET - : Kind.VARIABLE; + boolean secret = EnvironmentConfigFileSummary.looksLikeSecret(name, variable.getValue()); + String value = StringUtils.trimToNull(variable.getValue()); VariableAssignment existing = byName.get(name); if (existing == null) { byName.put( name, - new VariableAssignment(name, StringUtils.trimToNull(variable.getDescription()), kind)); + new VariableAssignment( + name, StringUtils.trimToNull(variable.getDescription()), secret, false, value)); return; } if (existing.description == null) { existing.description = StringUtils.trimToNull(variable.getDescription()); } - if (kind == Kind.SECRET) { - existing.kind = Kind.SECRET; + // Later files override, the same way configuration files are applied. + if (value != null) { + existing.defaultValue = value; + } + if (secret) { + existing.secret = true; } } @@ -187,19 +184,33 @@ public EnvironmentSource( } /** - * A variable to place in one of the three lists. {@code kind} is a suggestion until the user - * confirms it. The description may be updated while files are merged. + * One imported variable. {@code secret} and {@code mandatory} are suggestions until the user + * confirms them. The description and default may be updated while files are merged. A later + * non-blank value replaces the default. A blank value does not. */ @Getter public static final class VariableAssignment { private final String name; private String description; - private Kind kind; + private boolean secret; + private boolean mandatory; + private String defaultValue; + + public VariableAssignment(String name, String description, boolean secret) { + this(name, description, secret, false, null); + } + + public VariableAssignment(String name, String description, boolean secret, boolean mandatory) { + this(name, description, secret, mandatory, null); + } - public VariableAssignment(String name, String description, Kind kind) { + public VariableAssignment( + String name, String description, boolean secret, boolean mandatory, String defaultValue) { this.name = name; this.description = description; - this.kind = kind == null ? Kind.VARIABLE : kind; + this.secret = secret; + this.mandatory = mandatory; + this.defaultValue = StringUtils.trimToNull(defaultValue); } } } diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentMaterializer.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentMaterializer.java index 4cf90e88901..34d1f6d9dc3 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentMaterializer.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentMaterializer.java @@ -30,12 +30,14 @@ import org.apache.hop.core.vfs.HopVfs; import org.apache.hop.projects.config.ProjectsConfig; import org.apache.hop.projects.project.Project; +import org.apache.hop.projects.util.PathVariableReplacer; /** * Creates local lifecycle environments from the embedded definitions in a project. * - *

The configuration file for an environment receives mandatory variables and secrets only. - * Optional variables stay in the project and are applied when the environment is enabled. + *

The configuration file for an environment receives every variable marked mandatory or secret, + * in list order. A variable that is neither stays in the project and is applied when the + * environment is enabled. */ public final class EmbeddedEnvironmentMaterializer { @@ -88,14 +90,31 @@ public static String configFileName(String environmentName) { } /** - * @param folder folder that will hold the file + * @param folder folder that will hold the file. Variables must already be resolved. * @param environmentName environment name * @return path of {@code /.json} * @throws HopException when the folder cannot be resolved */ public static String configFilePath(String folder, String environmentName) throws HopException { + return configFilePath(null, folder, environmentName); + } + + /** + * Resolve variables in {@code folder} before asking VFS for the file. A folder such as {@code + * ${HOP_CONFIG_FOLDER}/environments/project} is absolute once the variable is resolved. Leaving + * the variable in the string makes VFS treat it as a path relative to the working directory. + * + * @param variables variable space, or null when {@code folder} is already resolved + * @param folder folder that will hold the file + * @param environmentName environment name + * @return resolved path of {@code /.json} + * @throws HopException when the folder cannot be resolved + */ + public static String configFilePath(IVariables variables, String folder, String environmentName) + throws HopException { String fileName = configFileName(environmentName); - try (FileObject parent = HopVfs.getFileObject(folder)) { + String resolvedFolder = resolvedFolder(variables, folder); + try (FileObject parent = HopVfs.getFileObject(resolvedFolder)) { FileObject file = parent.resolveFile(fileName); String scheme = file.getName().getScheme(); if (scheme == null || "file".equalsIgnoreCase(scheme)) { @@ -114,7 +133,28 @@ public static String configFilePath(String folder, String environmentName) throw * @return true when {@code folder} is the project home or a folder inside it */ public static boolean isInsideProjectHome(String folder, String projectHome) { - if (StringUtils.isBlank(folder) || StringUtils.isBlank(projectHome)) { + return isInsideProjectHome(null, folder, projectHome); + } + + /** + * Resolve both folders before comparing them. An unresolved {@code ${HOP_CONFIG_FOLDER}} would + * otherwise be compared as a path under the working directory. + * + * @param variables variable space, or null when both folders are already resolved + * @param folder candidate folder + * @param projectHome project home folder + * @return true when {@code folder} is the project home or a folder inside it + */ + public static boolean isInsideProjectHome( + IVariables variables, String folder, String projectHome) { + if (variables != null) { + folder = variables.resolve(folder); + projectHome = variables.resolve(projectHome); + } + if (StringUtils.isBlank(folder) + || StringUtils.isBlank(projectHome) + || folder.contains("${") + || projectHome.contains("${")) { return false; } try (FileObject home = HopVfs.getFileObject(projectHome); @@ -128,15 +168,19 @@ public static boolean isInsideProjectHome(String folder, String projectHome) { } /** - * Mandatory variables and secrets, in that order. Optional variables are not included. + * Variables marked mandatory or secret, in list order. A variable that is neither is left out. A + * secret is stored whether or not it is mandatory. * * @param environment embedded definition * @return variables to store in the local configuration file */ public static List variablesToStore(EmbeddedEnvironment environment) { List stored = new ArrayList<>(); - addVariables(stored, environment == null ? null : environment.getMandatoryVariables()); - addVariables(stored, environment == null ? null : environment.getSecretVariables()); + if (environment == null) { + return stored; + } + environment.absorbLegacyVariables(); + addVariables(stored, environment.getVariables()); return stored; } @@ -148,13 +192,36 @@ public static List variablesToStore(EmbeddedEnvironment envir * @param config environments already on this computer * @param project project that holds the definitions * @param projectName project the new environments belong to - * @param folder folder for the configuration files + * @param folder folder for the configuration files. Variables must already be resolved. * @return what was created and what was left alone * @throws HopException when a configuration file cannot be written */ public static MaterializeResult materialize( ProjectsConfig config, Project project, String projectName, String folder) throws HopException { + return materialize(null, config, project, projectName, folder); + } + + /** + * Same as {@link #materialize(ProjectsConfig, Project, String, String)}, resolving variables in + * {@code folder} before the file is written. The path stored on the lifecycle environment keeps a + * variable such as {@code ${HOP_CONFIG_FOLDER}} when one matches the resolved folder. + * + * @param variables variable space, or null when {@code folder} is already resolved + * @param config environments already on this computer + * @param project project that holds the definitions + * @param projectName project the new environments belong to + * @param folder folder for the configuration files + * @return what was created and what was left alone + * @throws HopException when a configuration file cannot be written + */ + public static MaterializeResult materialize( + IVariables variables, + ProjectsConfig config, + Project project, + String projectName, + String folder) + throws HopException { MaterializeResult result = new MaterializeResult(); if (project == null || project.getEmbeddedEnvironments() == null) { return result; @@ -170,13 +237,14 @@ public static MaterializeResult materialize( result.skippedExistingNames.add(embedded.getName()); continue; } - String path = configFilePath(folder, embedded.getName()); - if (!writeConfigFileIfMissing(path, embedded)) { + String resolvedPath = configFilePath(variables, folder, embedded.getName()); + if (!writeConfigFileIfMissing(resolvedPath, embedded)) { result.keptExistingFiles.add(embedded.getName()); } + String reference = configurationFileReference(variables, resolvedPath); LifecycleEnvironment environment = new LifecycleEnvironment( - embedded.getName(), "", projectName, new ArrayList<>(List.of(path))); + embedded.getName(), "", projectName, new ArrayList<>(List.of(reference))); environment.setEmbeddedEnvironmentName(embedded.getName()); result.created.add(environment); } @@ -185,7 +253,7 @@ public static MaterializeResult materialize( /** * @param path configuration file path - * @param embedded definition whose mandatory variables and secrets are written + * @param embedded definition whose mandatory and secret variables are written * @return true when a new file was written, false when a file was already there * @throws HopException when the file cannot be written */ @@ -217,6 +285,9 @@ private static void addVariables( if (variable == null || StringUtils.isBlank(variable.getName())) { continue; } + if (!variable.isMandatory() && !variable.isSecret()) { + continue; + } stored.add( new DescribedVariable( variable.getName().trim(), @@ -225,6 +296,33 @@ private static void addVariables( } } + /** + * Folder to pass to VFS. Variables are resolved first so {@code ${HOP_CONFIG_FOLDER}} is not + * treated as a relative directory name. + */ + private static String resolvedFolder(IVariables variables, String folder) throws HopException { + String resolved = variables == null || folder == null ? folder : variables.resolve(folder); + if (resolved != null && resolved.contains("${")) { + throw new HopException( + "Folder '" + + folder + + "' still contains a variable after resolution. The configuration file was not written."); + } + return resolved; + } + + /** + * Path recorded on the lifecycle environment. The file itself is written to {@code resolvedPath}. + * A matching path variable, for example {@code ${HOP_CONFIG_FOLDER}}, is put back so the + * reference stays the same kind of path the directory dialog returns. + */ + private static String configurationFileReference(IVariables variables, String resolvedPath) { + if (variables == null || StringUtils.isBlank(resolvedPath)) { + return resolvedPath; + } + return PathVariableReplacer.replacePathWithVariable(variables, resolvedPath); + } + private static String withTrailingSlash(String path) { String normalized = path.replace('\\', '/'); if (!normalized.endsWith("/")) { @@ -233,7 +331,7 @@ private static String withTrailingSlash(String path) { return normalized; } - /** Environments created by {@link #materialize} and the ones that were left alone. */ + /** Environments created by materialize and the ones that were left alone. */ @Getter public static final class MaterializeResult { private final List created = new ArrayList<>(); diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentValidator.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentValidator.java index 4a252bcb751..d516fe3b7fc 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentValidator.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentValidator.java @@ -38,11 +38,10 @@ public static void normalize(EmbeddedEnvironment environment) { if (environment == null) { return; } + environment.absorbLegacyVariables(); environment.setName(StringUtils.trimToNull(environment.getName())); environment.setDescription(StringUtils.trimToNull(environment.getDescription())); environment.setVariables(normalizeVariables(environment.getVariables())); - environment.setMandatoryVariables(normalizeVariables(environment.getMandatoryVariables())); - environment.setSecretVariables(normalizeVariables(environment.getSecretVariables())); } /** @@ -54,15 +53,15 @@ public static boolean missingName(EmbeddedEnvironment environment) { } /** - * @param environment definition whose three lists are checked together + * @param environment definition whose variables are checked * @return the first variable name that appears more than once, or null */ public static String duplicateVariableName(EmbeddedEnvironment environment) { - if (environment == null) { + if (environment == null || environment.getVariables() == null) { return null; } Set seen = new HashSet<>(); - for (EmbeddedEnvironmentVariable variable : allVariables(environment)) { + for (EmbeddedEnvironmentVariable variable : environment.getVariables()) { if (variable == null || StringUtils.isBlank(variable.getName())) { continue; } @@ -113,21 +112,6 @@ public static boolean nameTaken(String name, List otherNames) { return false; } - private static List allVariables(EmbeddedEnvironment environment) { - List all = new ArrayList<>(); - addAll(all, environment.getVariables()); - addAll(all, environment.getMandatoryVariables()); - addAll(all, environment.getSecretVariables()); - return all; - } - - private static void addAll( - List target, List source) { - if (source != null) { - target.addAll(source); - } - } - private static List normalizeVariables( List source) { List normalized = new ArrayList<>(); diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentVariable.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentVariable.java index 9f078724667..77ce5114f0e 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentVariable.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EmbeddedEnvironmentVariable.java @@ -23,8 +23,8 @@ import lombok.Setter; /** - * A variable declared on an embedded environment. {@code defaultValue} is a placeholder checked in - * with the project, not a live value. + * A variable declared on an embedded environment. {@code defaultValue} is checked in with the + * project. */ @Getter @Setter @@ -37,4 +37,19 @@ public class EmbeddedEnvironmentVariable { private String defaultValue; private String description; + + /** True when a person has to set this variable on their own computer. */ + private boolean mandatory; + + /** True when this value is a secret, such as a password or a token. */ + private boolean secret; + + public EmbeddedEnvironmentVariable(String name, String defaultValue, String description) { + this(name, defaultValue, description, false, false); + } + + public EmbeddedEnvironmentVariable( + String name, String defaultValue, String description, boolean mandatory) { + this(name, defaultValue, description, mandatory, false); + } } diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java index f4bedb6a19f..d6000a4a6fb 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java @@ -1657,7 +1657,7 @@ public static boolean offerEmbeddedEnvironments( if (projectConfig != null && variables != null) { projectHome = variables.resolve(projectConfig.getProjectHome()); } - if (EmbeddedEnvironmentMaterializer.isInsideProjectHome(folder, projectHome)) { + if (EmbeddedEnvironmentMaterializer.isInsideProjectHome(variables, folder, projectHome)) { MessageBox warning = new MessageBox(shell, SWT.YES | SWT.NO | SWT.ICON_WARNING); warning.setText( BaseMessages.getString( @@ -1673,7 +1673,8 @@ public static boolean offerEmbeddedEnvironments( try { ProjectsConfig config = ProjectsConfigSingleton.getConfig(); EmbeddedEnvironmentMaterializer.MaterializeResult result = - EmbeddedEnvironmentMaterializer.materialize(config, project, projectName, folder); + EmbeddedEnvironmentMaterializer.materialize( + variables, config, project, projectName, folder); for (LifecycleEnvironment created : result.getCreated()) { config.addEnvironment(created); } diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/EmbeddedEnvironmentDialog.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/EmbeddedEnvironmentDialog.java index 0e9cd054dc9..43cd5fca719 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/EmbeddedEnvironmentDialog.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/EmbeddedEnvironmentDialog.java @@ -40,13 +40,10 @@ import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; import org.apache.hop.ui.util.HelpUtils; import org.eclipse.swt.SWT; -import org.eclipse.swt.custom.CTabFolder; -import org.eclipse.swt.custom.CTabItem; import org.eclipse.swt.layout.FormAttachment; import org.eclipse.swt.layout.FormData; import org.eclipse.swt.layout.FormLayout; import org.eclipse.swt.widgets.Button; -import org.eclipse.swt.widgets.Composite; import org.eclipse.swt.widgets.Control; import org.eclipse.swt.widgets.Dialog; import org.eclipse.swt.widgets.Label; @@ -54,7 +51,7 @@ import org.eclipse.swt.widgets.TableItem; import org.eclipse.swt.widgets.Text; -/** Edits one embedded environment: name, description, and the three variable lists. */ +/** Edits one embedded environment: name, description, and variables. */ public class EmbeddedEnvironmentDialog extends Dialog { private static final Class PKG = EmbeddedEnvironmentDialog.class; @@ -67,8 +64,6 @@ public class EmbeddedEnvironmentDialog extends Dialog { private TextVar wName; private Text wDescription; private TableView wVariables; - private TableView wMandatoryVariables; - private TableView wSecretVariables; private EmbeddedEnvironment returnValue; @@ -89,6 +84,7 @@ public EmbeddedEnvironmentDialog( * constructor is updated only when the user confirms. */ public EmbeddedEnvironment open() { + environment.absorbLegacyVariables(); Shell parent = getParent(); shell = new Shell(parent, SWT.DIALOG_TRIM | SWT.APPLICATION_MODAL | SWT.RESIZE); shell.setImage( @@ -120,34 +116,18 @@ public EmbeddedEnvironment open() { Control last = addName(margin, middle); last = addDescription(margin, middle, last); - CTabFolder tabs = new CTabFolder(shell, SWT.BORDER); - PropsUi.setLook(tabs); - FormData fdTabs = new FormData(); - fdTabs.left = new FormAttachment(0, 0); - fdTabs.top = new FormAttachment(last, margin); - fdTabs.right = new FormAttachment(100, 0); - fdTabs.bottom = new FormAttachment(wOk, -margin * 2); - tabs.setLayoutData(fdTabs); - + int size = environment.getVariables() == null ? 0 : environment.getVariables().size(); wVariables = - addVariableTab( - tabs, - "EmbeddedEnvironmentDialog.Tab.Variables", - environment.getVariables(), - "EmbeddedEnvironmentDialog.Tab.Variables.Tooltip"); - wMandatoryVariables = - addVariableTab( - tabs, - "EmbeddedEnvironmentDialog.Tab.Mandatory", - environment.getMandatoryVariables(), - "EmbeddedEnvironmentDialog.Tab.Mandatory.Tooltip"); - wSecretVariables = - addVariableTab( - tabs, - "EmbeddedEnvironmentDialog.Tab.Secrets", - environment.getSecretVariables(), - "EmbeddedEnvironmentDialog.Tab.Secrets.Tooltip"); - tabs.setSelection(0); + new TableView( + variables, shell, SWT.BORDER, variableColumns(), Math.max(size, 3), event -> {}, props); + PropsUi.setLook(wVariables); + FormData fdVariables = new FormData(); + fdVariables.left = new FormAttachment(0, 0); + fdVariables.top = new FormAttachment(last, margin); + fdVariables.right = new FormAttachment(100, 0); + fdVariables.bottom = new FormAttachment(wOk, -margin * 2); + wVariables.setLayoutData(fdVariables); + fillVariables(wVariables, environment.getVariables()); wName.setText(Const.NVL(environment.getName(), "")); wDescription.setText(Const.NVL(environment.getDescription(), "")); @@ -201,35 +181,6 @@ private Control addDescription(int margin, int middle, Control previous) { return wDescription; } - private TableView addVariableTab( - CTabFolder folder, String tabKey, List rows, String tooltipKey) { - CTabItem tab = new CTabItem(folder, SWT.NONE); - tab.setText(BaseMessages.getString(PKG, tabKey)); - tab.setToolTipText(BaseMessages.getString(PKG, tooltipKey)); - Composite comp = new Composite(folder, SWT.NONE); - PropsUi.setLook(comp); - FormLayout layout = new FormLayout(); - layout.marginWidth = PropsUi.getFormMargin(); - layout.marginHeight = PropsUi.getFormMargin(); - comp.setLayout(layout); - tab.setControl(comp); - - int size = rows == null ? 0 : rows.size(); - ColumnInfo[] columnInfo = variableColumns(); - TableView table = - new TableView( - variables, comp, SWT.BORDER, columnInfo, Math.max(size, 3), event -> {}, props); - PropsUi.setLook(table); - FormData fdTable = new FormData(); - fdTable.left = new FormAttachment(0, 0); - fdTable.right = new FormAttachment(100, 0); - fdTable.top = new FormAttachment(0, 0); - fdTable.bottom = new FormAttachment(100, 0); - table.setLayoutData(fdTable); - fillVariables(table, rows); - return table; - } - private ColumnInfo[] variableColumns() { ColumnInfo[] columnInfo = new ColumnInfo[] { @@ -248,12 +199,28 @@ private ColumnInfo[] variableColumns() { ColumnInfo.COLUMN_TYPE_TEXT, false, false), + new ColumnInfo( + BaseMessages.getString(PKG, "EmbeddedEnvironmentDialog.Column.Mandatory"), + ColumnInfo.COLUMN_TYPE_CCOMBO, + new String[] {"Y", "N"}, + true), + new ColumnInfo( + BaseMessages.getString(PKG, "EmbeddedEnvironmentDialog.Column.Secret"), + ColumnInfo.COLUMN_TYPE_CCOMBO, + new String[] {"Y", "N"}, + true), }; columnInfo[0].setUsingVariables(true); columnInfo[0].setNamingSchemeType(NamingSchemeTypes.HOP_VARIABLE); columnInfo[1].setUsingVariables(true); columnInfo[1].setToolTip( BaseMessages.getString(PKG, "EmbeddedEnvironmentDialog.Column.Default.Tooltip")); + columnInfo[3].setUsingVariables(false); + columnInfo[3].setToolTip( + BaseMessages.getString(PKG, "EmbeddedEnvironmentDialog.Column.Mandatory.Tooltip")); + columnInfo[4].setUsingVariables(false); + columnInfo[4].setToolTip( + BaseMessages.getString(PKG, "EmbeddedEnvironmentDialog.Column.Secret.Tooltip")); return columnInfo; } @@ -267,6 +234,8 @@ private void fillVariables(TableView table, List ro item.setText(1, Const.NVL(variable.getName(), "")); item.setText(2, Const.NVL(variable.getDefaultValue(), "")); item.setText(3, Const.NVL(variable.getDescription(), "")); + item.setText(4, variable.isMandatory() ? "Y" : "N"); + item.setText(5, variable.isSecret() ? "Y" : "N"); } table.setRowNums(); table.optWidth(true); @@ -279,7 +248,13 @@ private List readVariables(TableView table) { if (StringUtils.isBlank(item.getText(1))) { continue; } - rows.add(new EmbeddedEnvironmentVariable(item.getText(1), item.getText(2), item.getText(3))); + rows.add( + new EmbeddedEnvironmentVariable( + item.getText(1), + item.getText(2), + item.getText(3), + "Y".equals(item.getText(4)), + "Y".equals(item.getText(5)))); } return rows; } @@ -289,8 +264,6 @@ private void ok() { edited.setName(wName.getText()); edited.setDescription(wDescription.getText()); edited.setVariables(readVariables(wVariables)); - edited.setMandatoryVariables(readVariables(wMandatoryVariables)); - edited.setSecretVariables(readVariables(wSecretVariables)); EmbeddedEnvironmentValidator.normalize(edited); if (EmbeddedEnvironmentValidator.missingName(edited)) { @@ -318,8 +291,6 @@ private void ok() { environment.setName(edited.getName()); environment.setDescription(edited.getDescription()); environment.setVariables(edited.getVariables()); - environment.setMandatoryVariables(edited.getMandatoryVariables()); - environment.setSecretVariables(edited.getSecretVariables()); returnValue = environment; dispose(); } diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ImportEmbeddedEnvironmentsDialog.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ImportEmbeddedEnvironmentsDialog.java index dd432272941..1d6f26a41a3 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ImportEmbeddedEnvironmentsDialog.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ImportEmbeddedEnvironmentsDialog.java @@ -26,7 +26,6 @@ import org.apache.hop.projects.environment.EmbeddedEnvironment; import org.apache.hop.projects.environment.EmbeddedEnvironmentImporter; import org.apache.hop.projects.environment.EmbeddedEnvironmentImporter.EnvironmentSource; -import org.apache.hop.projects.environment.EmbeddedEnvironmentImporter.Kind; import org.apache.hop.projects.environment.EmbeddedEnvironmentImporter.VariableAssignment; import org.apache.hop.projects.util.Defaults; import org.apache.hop.ui.core.ConstUi; @@ -49,8 +48,8 @@ import org.eclipse.swt.widgets.TableItem; /** - * Asks how each variable from the selected lifecycle environments should be stored: as a variable, - * as mandatory, or as a secret. Values from the configuration files are not shown or copied. + * Asks how each variable from the selected lifecycle environments should be stored. The value from + * the configuration file is copied in as the default and can be edited before it is saved. */ public class ImportEmbeddedEnvironmentsDialog extends Dialog { private static final Class PKG = ImportEmbeddedEnvironmentsDialog.class; @@ -163,7 +162,9 @@ private void ok() { List forSource = new ArrayList<>(); for (Row row : edited) { if (source.getName().equals(row.environmentName)) { - forSource.add(new VariableAssignment(row.name, row.description, row.kind)); + forSource.add( + new VariableAssignment( + row.name, row.description, row.secret, row.mandatory, row.defaultValue)); } } environments.add( @@ -184,8 +185,6 @@ private void dispose() { } private ColumnInfo[] columns() { - String[] kinds = - new String[] {kindLabel(Kind.VARIABLE), kindLabel(Kind.MANDATORY), kindLabel(Kind.SECRET)}; ColumnInfo environment = new ColumnInfo( BaseMessages.getString(PKG, "ImportEmbeddedEnvironmentsDialog.Column.Environment"), @@ -204,13 +203,34 @@ private ColumnInfo[] columns() { ColumnInfo.COLUMN_TYPE_TEXT, false, false); - ColumnInfo kind = + ColumnInfo defaultValue = new ColumnInfo( - BaseMessages.getString(PKG, "ImportEmbeddedEnvironmentsDialog.Column.Kind"), + BaseMessages.getString(PKG, "ImportEmbeddedEnvironmentsDialog.Column.Default"), + ColumnInfo.COLUMN_TYPE_TEXT, + false, + false); + defaultValue.setUsingVariables(true); + defaultValue.setToolTip( + BaseMessages.getString(PKG, "ImportEmbeddedEnvironmentsDialog.Column.Default.Tooltip")); + ColumnInfo mandatory = + new ColumnInfo( + BaseMessages.getString(PKG, "ImportEmbeddedEnvironmentsDialog.Column.Mandatory"), + ColumnInfo.COLUMN_TYPE_CCOMBO, + new String[] {"Y", "N"}, + true); + mandatory.setUsingVariables(false); + mandatory.setToolTip( + BaseMessages.getString(PKG, "ImportEmbeddedEnvironmentsDialog.Column.Mandatory.Tooltip")); + ColumnInfo secret = + new ColumnInfo( + BaseMessages.getString(PKG, "ImportEmbeddedEnvironmentsDialog.Column.Secret"), ColumnInfo.COLUMN_TYPE_CCOMBO, - kinds, + new String[] {"Y", "N"}, true); - return new ColumnInfo[] {environment, name, description, kind}; + secret.setUsingVariables(false); + secret.setToolTip( + BaseMessages.getString(PKG, "ImportEmbeddedEnvironmentsDialog.Column.Secret.Tooltip")); + return new ColumnInfo[] {environment, name, description, defaultValue, mandatory, secret}; } private void fillRows(List rows) { @@ -220,7 +240,9 @@ private void fillRows(List rows) { item.setText(1, Const.NVL(row.environmentName, "")); item.setText(2, Const.NVL(row.name, "")); item.setText(3, Const.NVL(row.description, "")); - item.setText(4, kindLabel(row.kind)); + item.setText(4, Const.NVL(row.defaultValue, "")); + item.setText(5, row.mandatory ? "Y" : "N"); + item.setText(6, row.secret ? "Y" : "N"); } wVariables.setRowNums(); wVariables.optWidth(true); @@ -238,7 +260,9 @@ private List readRows() { item.getText(1), item.getText(2), StringUtils.trimToNull(item.getText(3)), - kindOf(item.getText(4)))); + StringUtils.trimToNull(item.getText(4)), + "Y".equals(item.getText(5)), + "Y".equals(item.getText(6)))); } return rows; } @@ -252,24 +276,36 @@ private List rowsInOrder() { source.getName(), assignment.getName(), assignment.getDescription(), - assignment.getKind())); + assignment.getDefaultValue(), + assignment.isMandatory(), + assignment.isSecret())); } } return rows; } - /** One table row: the environment it belongs to and the list the user chose. */ + /** One table row: the environment it belongs to, and whether it is mandatory or a secret. */ private static final class Row { private final String environmentName; private final String name; private final String description; - private final Kind kind; + private final String defaultValue; + private final boolean mandatory; + private final boolean secret; - private Row(String environmentName, String name, String description, Kind kind) { + private Row( + String environmentName, + String name, + String description, + String defaultValue, + boolean mandatory, + boolean secret) { this.environmentName = environmentName; this.name = name; this.description = description; - this.kind = kind; + this.defaultValue = defaultValue; + this.mandatory = mandatory; + this.secret = secret; } } @@ -296,24 +332,4 @@ private String unreadableFiles() { } return text.toString(); } - - private static String kindLabel(Kind kind) { - Kind resolved = kind == null ? Kind.VARIABLE : kind; - return switch (resolved) { - case MANDATORY -> - BaseMessages.getString(PKG, "ImportEmbeddedEnvironmentsDialog.Kind.Mandatory"); - case SECRET -> BaseMessages.getString(PKG, "ImportEmbeddedEnvironmentsDialog.Kind.Secret"); - default -> BaseMessages.getString(PKG, "ImportEmbeddedEnvironmentsDialog.Kind.Variable"); - }; - } - - private static Kind kindOf(String label) { - if (kindLabel(Kind.MANDATORY).equals(label)) { - return Kind.MANDATORY; - } - if (kindLabel(Kind.SECRET).equals(label)) { - return Kind.SECRET; - } - return Kind.VARIABLE; - } } diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ParentProjectFolderSynchronizer.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ParentProjectFolderSynchronizer.java index 9f9ab226091..c990bf9e330 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ParentProjectFolderSynchronizer.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ParentProjectFolderSynchronizer.java @@ -165,13 +165,12 @@ static void synchronizeMapping( } copyTree(log, source, dest, "", mapping.isOverwrite(), exclusion, childConfigFile); - } catch (FileSystemException e) { + } catch (FileSystemException ignored) { logError( log, - "Parent folder '" + "Refused to copy parent folder '" + Const.NVL(mapping.getFolder(), "") - + "' is outside the project home (path traversal rejected)", - e); + + "' because it is outside the project home."); } finally { if (source != parentHome) { closeQuietly(source); @@ -243,14 +242,16 @@ static Pattern compileExclusion(ILogChannel log, ParentProjectFolder mapping) { try { return Pattern.compile(wildcard.trim()); } catch (PatternSyntaxException e) { + // getMessage() is several lines and looks like a stack trace in the build log. logError( log, - "Invalid exclusion regular expression '" + "Ignored invalid exclusion regular expression '" + wildcard + "' for parent folder '" + Const.NVL(mapping.getFolder(), "") - + "': " - + e.getMessage()); + + "' (" + + e.getDescription() + + "). That folder was not copied."); return null; } } diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/Project.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/Project.java index 9a43c55573c..cfff771257a 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/Project.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/Project.java @@ -207,6 +207,11 @@ public void readFromFile() throws HopException { project.embeddedEnvironments != null ? new ArrayList<>(project.embeddedEnvironments) : new ArrayList<>(); + for (EmbeddedEnvironment environment : this.embeddedEnvironments) { + if (environment != null) { + environment.absorbLegacyVariables(); + } + } } catch (Exception e) { throw new HopException( "Error saving project configuration to file '" + configFilename + "'", e); @@ -392,9 +397,8 @@ private void applyEmbeddedEnvironmentDefaults(IVariables variables, String envir + "' is not defined in this project. Continuing without those defaults."); return; } + embedded.absorbLegacyVariables(); applyEmbeddedVariables(variables, embedded.getVariables()); - applyEmbeddedVariables(variables, embedded.getMandatoryVariables()); - applyEmbeddedVariables(variables, embedded.getSecretVariables()); } private static void applyEmbeddedVariables( diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ProjectDialog.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ProjectDialog.java index 44dc65cd93b..a6cdcf63461 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ProjectDialog.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ProjectDialog.java @@ -49,6 +49,7 @@ import org.apache.hop.ui.core.PropsUi; import org.apache.hop.ui.core.dialog.BaseDialog; import org.apache.hop.ui.core.dialog.EnterSelectionDialog; +import org.apache.hop.ui.core.dialog.EnterStringDialog; import org.apache.hop.ui.core.dialog.ErrorDialog; import org.apache.hop.ui.core.dialog.MessageBox; import org.apache.hop.ui.core.gui.GuiResource; @@ -114,8 +115,10 @@ public class ProjectDialog extends Dialog { private TableView wEnvironments; private Button wAddEnvironment; private Button wEditEnvironment; + private Button wCopyEnvironment; private Button wDeleteEnvironment; private Button wImportEnvironment; + private Button wImplementEnvironment; /** Working copy of the embedded environments. Applied to the project on OK. */ private final List embeddedEnvironments = new ArrayList<>(); @@ -720,6 +723,11 @@ private void createEnvironmentsTab(CTabFolder folder, int margin) { wEditEnvironment = new Button(comp, SWT.PUSH); wEditEnvironment.setText(BaseMessages.getString(PKG, "ProjectDialog.Button.EditEnvironment")); wEditEnvironment.addListener(SWT.Selection, event -> editEmbeddedEnvironment()); + wCopyEnvironment = new Button(comp, SWT.PUSH); + wCopyEnvironment.setText(BaseMessages.getString(PKG, "ProjectDialog.Button.CopyEnvironment")); + wCopyEnvironment.setToolTipText( + BaseMessages.getString(PKG, "ProjectDialog.Button.CopyEnvironment.Tooltip")); + wCopyEnvironment.addListener(SWT.Selection, event -> copyEmbeddedEnvironment()); wDeleteEnvironment = new Button(comp, SWT.PUSH); wDeleteEnvironment.setText( BaseMessages.getString(PKG, "ProjectDialog.Button.DeleteEnvironment")); @@ -730,9 +738,22 @@ private void createEnvironmentsTab(CTabFolder folder, int margin) { wImportEnvironment.setToolTipText( BaseMessages.getString(PKG, "ProjectDialog.Button.ImportEnvironment.Tooltip")); wImportEnvironment.addListener(SWT.Selection, event -> importEmbeddedEnvironments()); + wImplementEnvironment = new Button(comp, SWT.PUSH); + wImplementEnvironment.setText( + BaseMessages.getString(PKG, "ProjectDialog.Button.ImplementEnvironment")); + wImplementEnvironment.setToolTipText( + BaseMessages.getString(PKG, "ProjectDialog.Button.ImplementEnvironment.Tooltip")); + wImplementEnvironment.addListener(SWT.Selection, event -> implementEmbeddedEnvironments()); BaseTransformDialog.positionBottomButtons( comp, - new Button[] {wAddEnvironment, wEditEnvironment, wDeleteEnvironment, wImportEnvironment}, + new Button[] { + wAddEnvironment, + wEditEnvironment, + wCopyEnvironment, + wDeleteEnvironment, + wImportEnvironment, + wImplementEnvironment + }, margin, null); @@ -827,6 +848,123 @@ private void deleteEmbeddedEnvironment() { needingProjectRefresh = true; } + private void copyEmbeddedEnvironment() { + if (!environmentsEditable()) { + return; + } + int index = wEnvironments.getSelectionIndex(); + if (index < 0 || index >= embeddedEnvironments.size()) { + return; + } + EmbeddedEnvironment selected = embeddedEnvironments.get(index); + String sourceName = Const.NVL(selected.getName(), ""); + String typed = suggestedCopyName(sourceName); + while (true) { + EnterStringDialog dialog = + new EnterStringDialog( + shell, + typed, + BaseMessages.getString(PKG, "ProjectDialog.Environments.Copy.Header"), + BaseMessages.getString(PKG, "ProjectDialog.Environments.Copy.Message", sourceName)); + dialog.setMandatory(true); + typed = dialog.open(); + if (typed == null) { + return; + } + String name = typed.trim(); + if (name.isEmpty()) { + showEnvironmentError( + "ProjectDialog.Environments.MissingName.Header", + "ProjectDialog.Environments.MissingName.Message"); + typed = ""; + continue; + } + if (EmbeddedEnvironmentValidator.nameTaken(name, environmentNamesExcept(-1))) { + showEnvironmentError( + "ProjectDialog.Environments.DuplicateName.Header", + "ProjectDialog.Environments.DuplicateName.Message", + name); + typed = name; + continue; + } + EmbeddedEnvironment copy = selected.copy(); + copy.setName(name); + embeddedEnvironments.add(copy); + refreshEmbeddedEnvironments(embeddedEnvironments.size() - 1); + needingProjectRefresh = true; + return; + } + } + + /** A name that is not already used, starting from {@code sourceName + " 2"}. */ + private String suggestedCopyName(String sourceName) { + String base = StringUtils.defaultIfBlank(StringUtils.trimToEmpty(sourceName), "environment"); + int suffix = 2; + String candidate = base + " " + suffix; + while (EmbeddedEnvironmentValidator.nameTaken(candidate, environmentNamesExcept(-1))) { + suffix++; + candidate = base + " " + suffix; + } + return candidate; + } + + /** + * Create local lifecycle environments from the definitions in this dialog. This is the same + * question and the same files as when a project with an existing configuration is added. + */ + private void implementEmbeddedEnvironments() { + if (!validateEmbeddedEnvironments()) { + return; + } + String projectName = StringUtils.trimToEmpty(wName.getText()); + if (projectName.isEmpty()) { + showEnvironmentError( + "ProjectDialog.Environments.Implement.MissingProjectName.Header", + "ProjectDialog.Environments.Implement.MissingProjectName.Message"); + return; + } + + Project implementing = new Project(); + List environments = new ArrayList<>(); + for (EmbeddedEnvironment environment : embeddedEnvironments) { + EmbeddedEnvironment copy = environment.copy(); + EmbeddedEnvironmentValidator.normalize(copy); + if (!EmbeddedEnvironmentValidator.missingName(copy)) { + environments.add(copy); + } + } + if (environments.isEmpty()) { + MessageBox box = new MessageBox(shell, SWT.OK | SWT.ICON_INFORMATION); + box.setText(BaseMessages.getString(PKG, "ProjectDialog.Environments.Implement.None.Header")); + box.setMessage( + BaseMessages.getString(PKG, "ProjectDialog.Environments.Implement.None.Message")); + box.open(); + return; + } + implementing.setEmbeddedEnvironments(environments); + + ProjectConfig config = + new ProjectConfig(projectName, sanitizePath(wHome.getText()), wConfigFile.getText()); + Variables space = new Variables(); + space.initializeFrom(HopGui.getInstance().getVariables()); + List shownVariables = new ArrayList<>(); + for (int i = 0; i < wVariables.nrNonEmpty(); i++) { + TableItem item = wVariables.getNonEmpty(i); + if (StringUtils.isNotEmpty(item.getText(1))) { + shownVariables.add(item.getText(1)); + space.setVariable(item.getText(1), Const.NVL(item.getText(2), "")); + } + } + // Opening the dialog copied the saved project variables into Hop GUI. Drop any that the table + // no longer lists, so a changed ENVIRONMENTS_FOLDER is the one that is used. + for (DescribedVariable variable : project.getDescribedVariables()) { + if (variable.getName() != null && !shownVariables.contains(variable.getName())) { + space.setVariable(variable.getName(), null); + } + } + ProjectsGuiPlugin.offerEmbeddedEnvironments(shell, implementing, config, space); + } + private void importEmbeddedEnvironments() { if (!environmentsEditable()) { return; @@ -1042,6 +1180,7 @@ private void updateReadOnlyWidgets() { if (wAddEnvironment != null) { wAddEnvironment.setEnabled(editable); wEditEnvironment.setEnabled(editable); + wCopyEnvironment.setEnabled(editable); wDeleteEnvironment.setEnabled(editable); wImportEnvironment.setEnabled(editable); } diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/util/ProjectsUtil.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/util/ProjectsUtil.java index 2d68d601812..806aae88d32 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/util/ProjectsUtil.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/util/ProjectsUtil.java @@ -476,9 +476,14 @@ public static List saveProjectConfig( try { saver.save(); } catch (HopException saveException) { + // The rename is already undone in memory. The thrown exception carries the original + // failure, so the log only records that this extra save was caught. log.logError( - "Unable to save the restored registration of project '" + currentName + "'", - saveException); + "Caught an error saving the restored registration of project '" + + currentName + + "' (" + + describeCause(saveException) + + ")."); } String message = notRestored.isEmpty() @@ -564,13 +569,16 @@ private static List restoreParentProjectName( child.project().setParentProjectName(parentProjectName); child.project().saveToFile(); } catch (Exception e) { + // The caller reports the projects that could not be restored. Log that the error was + // caught, without the stack trace. log.logError( - "Unable to restore parent project '" + "Caught an error restoring parent project '" + parentProjectName + "' of project '" + child.name() - + "'", - e); + + "' (" + + describeCause(e) + + ")."); notRestored.add(child.name()); } } @@ -602,9 +610,13 @@ private static Project loadProject( try { return projectConfig.loadProject(variables); } catch (Exception e) { + // A project that cannot be read is skipped. The stack trace is not useful here. log.logError( - "Unable to load project '" + projectConfig.getProjectName() + "' from its configuration", - e); + "Caught an error loading project '" + + projectConfig.getProjectName() + + "' (" + + describeCause(e) + + "). That project was skipped."); return null; } } diff --git a/plugins/misc/projects/src/main/resources/org/apache/hop/projects/gui/messages/messages_en_US.properties b/plugins/misc/projects/src/main/resources/org/apache/hop/projects/gui/messages/messages_en_US.properties index 096ffaa76fa..f3e9d160fe5 100644 --- a/plugins/misc/projects/src/main/resources/org/apache/hop/projects/gui/messages/messages_en_US.properties +++ b/plugins/misc/projects/src/main/resources/org/apache/hop/projects/gui/messages/messages_en_US.properties @@ -107,7 +107,7 @@ ProjectGuiPlugin.Lifecycle.Dialog.Header=Create project lifecycle environment? ProjectGuiPlugin.Lifecycle.Dialog.Message1=If this project is part of a lifecyle then perhaps you want to add it to a lifecycle environment? ProjectGuiPlugin.Lifecycle.Dialog.Message2=With it you can manage the specific settings like hostnames and paths for the environment ProjectGuiPlugin.EmbeddedEnvironments.Question.Header=Create lifecycle environments? -ProjectGuiPlugin.EmbeddedEnvironments.Question.Message=Project ''{0}'' defines lifecycle environments: {1}.\n\nCreate them on this computer? Required variables and secrets are written to a configuration file. Other variables keep the defaults from the project. +ProjectGuiPlugin.EmbeddedEnvironments.Question.Message=Project ''{0}'' defines lifecycle environments: {1}.\n\nCreate them on this computer? Mandatory variables and secrets are written to a configuration file with their default values. Other variables stay in the project. ProjectGuiPlugin.EmbeddedEnvironments.Question.Folder=Configuration files will be written to ''{0}''. ProjectGuiPlugin.EmbeddedEnvironments.Folder.Message=Choose the folder for the configuration files. Set the project variable '${ENVIRONMENTS_FOLDER}' to reuse that folder the next time someone adds this project. A folder outside the project keeps secrets out of version control. ProjectGuiPlugin.EmbeddedEnvironments.InsideProject.Header=Folder is inside the project diff --git a/plugins/misc/projects/src/main/resources/org/apache/hop/projects/project/messages/messages_en_US.properties b/plugins/misc/projects/src/main/resources/org/apache/hop/projects/project/messages/messages_en_US.properties index 170765c7d92..dd754f838b5 100644 --- a/plugins/misc/projects/src/main/resources/org/apache/hop/projects/project/messages/messages_en_US.properties +++ b/plugins/misc/projects/src/main/resources/org/apache/hop/projects/project/messages/messages_en_US.properties @@ -88,32 +88,44 @@ ProjectRename.Blocked.Unreadable=- ''{0}'': its configuration can''t be read, so ProjectRename.SaveFailed.Message=Project ''{0}'' wasn''t renamed to ''{1}'': the parent project of ''{2}'' couldn''t be saved. The rename was undone. ProjectRename.SaveFailed.NotRestored=Project ''{0}'' wasn''t renamed to ''{1}'': the parent project of ''{2}'' couldn''t be saved. The rename was undone, but these projects couldn''t be restored and still use ''{1}'' as their parent project: {3} ProjectDialog.Tab.Environments=Environments -ProjectDialog.Environments.Explanation=These definitions are stored in the project configuration and can be committed. Enter placeholder default values, not real passwords, hosts, or paths. Configuration files are not stored here. +ProjectDialog.Environments.Explanation=These definitions are stored in the project configuration and can be committed. A default that is the same on every computer can be the real value. Clear a password before you commit it. Configuration files are not stored here. ProjectDialog.Environments.Column.Name=Name ProjectDialog.Environments.Column.Description=Description ProjectDialog.Button.AddEnvironment=Add ProjectDialog.Button.EditEnvironment=Edit... +ProjectDialog.Button.CopyEnvironment=Copy... +ProjectDialog.Button.CopyEnvironment.Tooltip=Copy the selected environment. Only the name is asked for. +ProjectDialog.Environments.Copy.Header=Copy environment +ProjectDialog.Environments.Copy.Message=Name for the copy of ''{0}'' ProjectDialog.Button.DeleteEnvironment=Delete ProjectDialog.Button.ImportEnvironment=Import... ProjectDialog.Button.ImportEnvironment.Tooltip=Import one or more lifecycle environments of this project +ProjectDialog.Button.ImplementEnvironment=Implement +ProjectDialog.Button.ImplementEnvironment.Tooltip=Create the lifecycle environments on this computer from these definitions. Mandatory variables and secrets are written to a configuration file with their default values. +ProjectDialog.Environments.Implement.None.Header=No environments +ProjectDialog.Environments.Implement.None.Message=This project has no environment definitions to create. +ProjectDialog.Environments.Implement.MissingProjectName.Header=Project name +ProjectDialog.Environments.Implement.MissingProjectName.Message=Give the project a name before creating its environments. ProjectDialog.Environments.Import.None.Header=No environments ProjectDialog.Environments.Import.None.Message=This project has no lifecycle environments in the Hop configuration. ProjectDialog.Environments.Import.Select.Header=Import environments -ProjectDialog.Environments.Import.Select.Message=Select the lifecycle environments of this project to import. Their configuration files supply the variable names. +ProjectDialog.Environments.Import.Select.Message=Select the lifecycle environments of this project to import. Their configuration files supply the variable names and default values. ProjectDialog.Environments.Import.Replace.Header=Replace environment definitions? ProjectDialog.Environments.Import.Replace.Message=These environment definitions already exist in the project:\n{0}\n\nReplace them? Definitions that are not in this list are still imported. ProjectDialog.Environments.Import.Error.Header=Error ProjectDialog.Environments.Import.Error.Message=The environments could not be imported. ImportEmbeddedEnvironmentsDialog.Shell.Name=Import environment definitions -ImportEmbeddedEnvironmentsDialog.Explanation=Values from the configuration files are not copied. Mark each variable as a variable, mandatory, or a secret. You can set placeholder defaults afterwards. +ImportEmbeddedEnvironmentsDialog.Explanation=Values from the configuration files are copied in as defaults. Change or clear a default before saving when it should not be checked in with the project. Mark a variable as a secret when it is a password or token, and mark it mandatory when a person has to set it. ImportEmbeddedEnvironmentsDialog.Unreadable=These configuration files could not be read. Their variables are not listed. ImportEmbeddedEnvironmentsDialog.Column.Environment=Environment ImportEmbeddedEnvironmentsDialog.Column.Name=Name ImportEmbeddedEnvironmentsDialog.Column.Description=Description -ImportEmbeddedEnvironmentsDialog.Column.Kind=Use as -ImportEmbeddedEnvironmentsDialog.Kind.Variable=Variable -ImportEmbeddedEnvironmentsDialog.Kind.Mandatory=Mandatory -ImportEmbeddedEnvironmentsDialog.Kind.Secret=Secret +ImportEmbeddedEnvironmentsDialog.Column.Default=Default value +ImportEmbeddedEnvironmentsDialog.Column.Default.Tooltip=Copied from the configuration file. Change or clear it before saving when this value should not be checked in with the project. +ImportEmbeddedEnvironmentsDialog.Column.Mandatory=Mandatory +ImportEmbeddedEnvironmentsDialog.Column.Mandatory.Tooltip=Y when a person has to set this value on their own computer +ImportEmbeddedEnvironmentsDialog.Column.Secret=Secret +ImportEmbeddedEnvironmentsDialog.Column.Secret.Tooltip=Y for a password, token, or similar value. A secret is written to the configuration file on this computer. ProjectDialog.Environments.Delete.Header=Delete environment? ProjectDialog.Environments.Delete.Message=Delete the environment definition ''{0}'' from this project? ProjectDialog.Environments.MissingName.Header=Environment name @@ -121,23 +133,21 @@ ProjectDialog.Environments.MissingName.Message=Give the environment a name. ProjectDialog.Environments.DuplicateName.Header=Environment name ProjectDialog.Environments.DuplicateName.Message=An environment named ''{0}'' is already defined in this project. ProjectDialog.Environments.DuplicateVariable.Header=Variable name -ProjectDialog.Environments.DuplicateVariable.Message=Variable ''{0}'' is listed more than once. A name can be in only one of the three lists. +ProjectDialog.Environments.DuplicateVariable.Message=Variable ''{0}'' is listed more than once. EmbeddedEnvironmentDialog.Shell.Name=Environment definition EmbeddedEnvironmentDialog.Label.Name=Name EmbeddedEnvironmentDialog.Label.Description=Description -EmbeddedEnvironmentDialog.Tab.Variables=Variables -EmbeddedEnvironmentDialog.Tab.Variables.Tooltip=Optional variables. The default is a placeholder and is not a value for one computer. -EmbeddedEnvironmentDialog.Tab.Mandatory=Mandatory -EmbeddedEnvironmentDialog.Tab.Mandatory.Tooltip=Variables a person has to fill in on their own computer. The default is a placeholder. -EmbeddedEnvironmentDialog.Tab.Secrets=Secrets -EmbeddedEnvironmentDialog.Tab.Secrets.Tooltip=Secret variables. Store a placeholder such as change-to-your-password, not the real secret. EmbeddedEnvironmentDialog.Column.Name=Name EmbeddedEnvironmentDialog.Column.Default=Default value -EmbeddedEnvironmentDialog.Column.Default.Tooltip=Placeholder checked in with the project, for example change-to-your-password or specify the folder on the system where the files are. Not a real value. +EmbeddedEnvironmentDialog.Column.Default.Tooltip=Value checked in with the project. Use a placeholder such as change-to-your-password when the real value differs per computer. EmbeddedEnvironmentDialog.Column.Description=Description +EmbeddedEnvironmentDialog.Column.Mandatory=Mandatory +EmbeddedEnvironmentDialog.Column.Mandatory.Tooltip=Y when a person has to set this value on their own computer. +EmbeddedEnvironmentDialog.Column.Secret=Secret +EmbeddedEnvironmentDialog.Column.Secret.Tooltip=Y for a password, token, or similar value. A secret is written to the configuration file on this computer. EmbeddedEnvironmentDialog.MissingName.Header=Environment name EmbeddedEnvironmentDialog.MissingName.Message=Give the environment a name. EmbeddedEnvironmentDialog.DuplicateVariable.Header=Variable name -EmbeddedEnvironmentDialog.DuplicateVariable.Message=Variable ''{0}'' is listed more than once. A name can be in only one of the three lists. +EmbeddedEnvironmentDialog.DuplicateVariable.Message=Variable ''{0}'' is listed more than once. EmbeddedEnvironmentDialog.DuplicateName.Header=Environment name EmbeddedEnvironmentDialog.DuplicateName.Message=An environment named ''{0}'' is already defined in this project. diff --git a/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentImporterTest.java b/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentImporterTest.java index aeb18f791d3..a4544e9cac0 100644 --- a/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentImporterTest.java +++ b/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentImporterTest.java @@ -18,6 +18,7 @@ package org.apache.hop.projects.environment; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -34,7 +35,6 @@ import org.apache.hop.core.variables.IVariables; import org.apache.hop.core.variables.Variables; import org.apache.hop.projects.environment.EmbeddedEnvironmentImporter.EnvironmentSource; -import org.apache.hop.projects.environment.EmbeddedEnvironmentImporter.Kind; import org.apache.hop.projects.environment.EmbeddedEnvironmentImporter.VariableAssignment; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -60,7 +60,7 @@ void tearDown() throws Exception { } @Test - void readClassifiesSecretsAndDropsValues() throws Exception { + void readCopiesConfigurationValuesAsDefaults() throws Exception { String configPath = writeConfig("dev.json", variables()); LifecycleEnvironment environment = new LifecycleEnvironment("dev", "Developer workstation", "warehouse", List.of(configPath)); @@ -78,22 +78,35 @@ void readClassifiesSecretsAndDropsValues() throws Exception { assertEquals("Developer workstation", source.getDescription()); assertTrue(source.getUnreadableFiles().isEmpty()); assertEquals(List.of("LOG_LEVEL", "DB_HOST", "DB_PASSWORD", "DB_TOKEN"), names(source)); - assertEquals(Kind.VARIABLE, kind(source, "LOG_LEVEL")); - assertEquals(Kind.VARIABLE, kind(source, "DB_HOST")); - assertEquals(Kind.SECRET, kind(source, "DB_PASSWORD")); - assertEquals(Kind.SECRET, kind(source, "DB_TOKEN")); + assertFalse(secret(source, "LOG_LEVEL")); + assertFalse(secret(source, "DB_HOST")); + assertTrue(secret(source, "DB_PASSWORD")); + assertTrue(secret(source, "DB_TOKEN")); assertEquals("Hop log level", description(source, "LOG_LEVEL")); + assertEquals("Basic", defaultValue(source, "LOG_LEVEL")); + assertEquals("db.example", defaultValue(source, "DB_HOST")); + assertEquals("real-password", defaultValue(source, "DB_PASSWORD")); + assertEquals(Encr.PASSWORD_ENCRYPTED_PREFIX + "abc", defaultValue(source, "DB_TOKEN")); EmbeddedEnvironment embedded = EmbeddedEnvironmentImporter.toEmbeddedEnvironment( source.getName(), source.getDescription(), source.getVariables()); - assertNull(embedded.getVariables().get(0).getDefaultValue()); + assertEquals(4, embedded.getVariables().size()); + assertEquals("Basic", embedded.getVariables().get(0).getDefaultValue()); assertEquals("LOG_LEVEL", embedded.getVariables().get(0).getName()); + assertFalse(embedded.getVariables().get(0).isSecret()); + assertEquals("db.example", embedded.getVariables().get(1).getDefaultValue()); assertEquals("DB_HOST", embedded.getVariables().get(1).getName()); - assertEquals("DB_PASSWORD", embedded.getSecretVariables().get(0).getName()); - assertNull(embedded.getSecretVariables().get(0).getDefaultValue()); - assertEquals("DB_TOKEN", embedded.getSecretVariables().get(1).getName()); - assertTrue(embedded.getMandatoryVariables().isEmpty()); + assertFalse(embedded.getVariables().get(1).isSecret()); + assertEquals("real-password", embedded.getVariables().get(2).getDefaultValue()); + assertEquals("DB_PASSWORD", embedded.getVariables().get(2).getName()); + assertTrue(embedded.getVariables().get(2).isSecret()); + assertEquals( + Encr.PASSWORD_ENCRYPTED_PREFIX + "abc", embedded.getVariables().get(3).getDefaultValue()); + assertEquals("DB_TOKEN", embedded.getVariables().get(3).getName()); + assertTrue(embedded.getVariables().get(3).isSecret()); + assertFalse(embedded.getVariables().get(1).isMandatory()); + assertFalse(embedded.getVariables().get(2).isMandatory()); } @Test @@ -101,12 +114,17 @@ void readMergesFilesAndKeepsAnUnreadablePath() throws Exception { IVariables variables = new Variables(); variables.setVariable("ENV_DIR", tempRoot.toString()); String first = - writeConfig("first.json", List.of(new DescribedVariable("DB_HOST", "db.example", ""))); + writeConfig( + "first.json", + List.of( + new DescribedVariable("DB_HOST", "db.example", ""), + new DescribedVariable("DATA_FOLDER", " /data ", "Data folder"))); String second = writeConfig( "second.json", List.of( new DescribedVariable("DB_HOST", "other", "JDBC host"), + new DescribedVariable("DATA_FOLDER", " ", "later description"), new DescribedVariable( "API_KEY", Encr.PASSWORD_ENCRYPTED_PREFIX + "abc", "API key"))); LifecycleEnvironment environment = @@ -118,23 +136,31 @@ void readMergesFilesAndKeepsAnUnreadablePath() throws Exception { assertNull(source.getDescription()); assertEquals(List.of("${ENV_DIR}/missing.json"), source.getUnreadableFiles()); - assertEquals(Kind.VARIABLE, kind(source, "DB_HOST")); + assertFalse(secret(source, "DB_HOST")); assertEquals("JDBC host", description(source, "DB_HOST")); - assertEquals(Kind.SECRET, kind(source, "API_KEY")); + assertEquals("other", defaultValue(source, "DB_HOST")); + assertEquals("Data folder", description(source, "DATA_FOLDER")); + assertEquals("/data", defaultValue(source, "DATA_FOLDER")); + assertTrue(secret(source, "API_KEY")); + assertEquals(Encr.PASSWORD_ENCRYPTED_PREFIX + "abc", defaultValue(source, "API_KEY")); EmbeddedEnvironment embedded = EmbeddedEnvironmentImporter.toEmbeddedEnvironment( source.getName(), source.getDescription(), List.of( - new VariableAssignment("DB_HOST", "JDBC host", Kind.MANDATORY), - new VariableAssignment("API_KEY", "API key", Kind.SECRET), - new VariableAssignment(" ", "ignored", Kind.VARIABLE))); - assertEquals(1, embedded.getMandatoryVariables().size()); - assertEquals("DB_HOST", embedded.getMandatoryVariables().get(0).getName()); - assertNull(embedded.getMandatoryVariables().get(0).getDefaultValue()); - assertEquals("API_KEY", embedded.getSecretVariables().get(0).getName()); - assertTrue(embedded.getVariables().isEmpty()); + new VariableAssignment("DB_HOST", "JDBC host", false, true, " other "), + new VariableAssignment("API_KEY", "API key", true, true), + new VariableAssignment(" ", "ignored", false))); + assertEquals(2, embedded.getVariables().size()); + assertEquals("DB_HOST", embedded.getVariables().get(0).getName()); + assertEquals("other", embedded.getVariables().get(0).getDefaultValue()); + assertTrue(embedded.getVariables().get(0).isMandatory()); + assertFalse(embedded.getVariables().get(0).isSecret()); + assertEquals("API_KEY", embedded.getVariables().get(1).getName()); + assertNull(embedded.getVariables().get(1).getDefaultValue()); + assertTrue(embedded.getVariables().get(1).isMandatory()); + assertTrue(embedded.getVariables().get(1).isSecret()); } private List variables() { @@ -162,14 +188,18 @@ private static List names(EnvironmentSource source) { return names; } - private static Kind kind(EnvironmentSource source, String name) { - return assignment(source, name).getKind(); + private static boolean secret(EnvironmentSource source, String name) { + return assignment(source, name).isSecret(); } private static String description(EnvironmentSource source, String name) { return assignment(source, name).getDescription(); } + private static String defaultValue(EnvironmentSource source, String name) { + return assignment(source, name).getDefaultValue(); + } + private static VariableAssignment assignment(EnvironmentSource source, String name) { for (VariableAssignment assignment : source.getVariables()) { if (name.equals(assignment.getName())) { diff --git a/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentMaterializerTest.java b/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentMaterializerTest.java index f30572e8e1e..34db55c7a7a 100644 --- a/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentMaterializerTest.java +++ b/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentMaterializerTest.java @@ -127,6 +127,42 @@ void configFilePathDoesNotEscapeTheFolder() throws Exception { assertTrue(path.contains(tempRoot.getFileName().toString())); } + @Test + void configFilePathResolvesVariablesBeforeBuildingTheFilename() throws Exception { + Path config = tempRoot.resolve("config"); + Path expected = + config + .resolve("environments") + .resolve("retail-example") + .resolve("retail-example-docker-postgres.json"); + IVariables variables = new Variables(); + variables.setVariable("HOP_CONFIG_FOLDER", config.toString()); + + String path = + EmbeddedEnvironmentMaterializer.configFilePath( + variables, + "${HOP_CONFIG_FOLDER}/environments/retail-example/", + "retail-example-docker-postgres"); + + try (FileObject expectedFile = HopVfs.getFileObject(expected.toString())) { + assertEquals(expectedFile.getName().getPath(), path); + } + assertFalse(path.contains("${")); + assertFalse(path.contains(new File(System.getProperty("user.dir")).getName() + "/${")); + } + + @Test + void configFilePathRefusesAFolderWhoseVariableDoesNotResolve() { + IVariables variables = new Variables(); + HopException error = + assertThrows( + HopException.class, + () -> + EmbeddedEnvironmentMaterializer.configFilePath( + variables, "${HOP_CONFIG_FOLDER}/environments/retail-example", "dev")); + assertTrue(error.getMessage().contains("still contains a variable")); + } + @Test void insideProjectHomeMatchesTheHomeAndItsChildren() throws Exception { Path home = tempRoot.resolve("proj"); @@ -145,18 +181,46 @@ void insideProjectHomeMatchesTheHomeAndItsChildren() throws Exception { assertFalse(EmbeddedEnvironmentMaterializer.isInsideProjectHome(child.toString(), " ")); } + @Test + void insideProjectHomeResolvesTheFolder() throws Exception { + Path home = tempRoot.resolve("proj"); + Path config = tempRoot.resolve("config"); + Path child = home.resolve("environments"); + Files.createDirectories(child); + Files.createDirectories(config); + IVariables variables = new Variables(); + variables.setVariable("HOP_CONFIG_FOLDER", config.toString()); + variables.setVariable("PROJECT_HOME", home.toString()); + + assertFalse( + EmbeddedEnvironmentMaterializer.isInsideProjectHome( + variables, + "${HOP_CONFIG_FOLDER}/environments/retail-example", + System.getProperty("user.dir"))); + assertTrue( + EmbeddedEnvironmentMaterializer.isInsideProjectHome( + variables, "${PROJECT_HOME}/environments", "${PROJECT_HOME}")); + assertFalse( + EmbeddedEnvironmentMaterializer.isInsideProjectHome( + variables, "${MISSING}/environments", home.toString())); + } + @Test void variablesToStoreKeepsMandatoryAndSecretsOnly() { EmbeddedEnvironment environment = sampleEnvironment("dev"); + environment + .getVariables() + .add(new EmbeddedEnvironmentVariable("API_TOKEN", "change-me", "token", false, true)); List stored = EmbeddedEnvironmentMaterializer.variablesToStore(environment); - assertEquals(2, stored.size()); + assertEquals(3, stored.size()); assertEquals("DB_HOST", stored.get(0).getName()); assertEquals("specify the database host", stored.get(0).getValue()); assertEquals("JDBC host", stored.get(0).getDescription()); assertEquals("DB_PASSWORD", stored.get(1).getName()); assertEquals("change-to-your-password", stored.get(1).getValue()); assertEquals("JDBC password", stored.get(1).getDescription()); + assertEquals("API_TOKEN", stored.get(2).getName()); assertTrue(EmbeddedEnvironmentMaterializer.variablesToStore(null).isEmpty()); } @@ -164,8 +228,8 @@ void variablesToStoreKeepsMandatoryAndSecretsOnly() { void materializeWritesMandatoryAndSecrets() throws Exception { EmbeddedEnvironment environment = sampleEnvironment("dev"); environment - .getMandatoryVariables() - .add(new EmbeddedEnvironmentVariable("EMPTY_DEFAULT", null, null)); + .getVariables() + .add(new EmbeddedEnvironmentVariable("EMPTY_DEFAULT", null, null, true)); Project project = new Project(); project.getEmbeddedEnvironments().add(environment); project.getEmbeddedEnvironments().add(new EmbeddedEnvironment()); @@ -197,11 +261,13 @@ void materializeWritesMandatoryAndSecrets() throws Exception { List stored = configFile.getDescribedVariables(); assertEquals(3, stored.size()); assertEquals("DB_HOST", stored.get(0).getName()); - assertEquals("EMPTY_DEFAULT", stored.get(1).getName()); - assertEquals("", stored.get(1).getValue()); - assertEquals("", stored.get(1).getDescription()); - assertEquals("DB_PASSWORD", stored.get(2).getName()); - assertEquals("change-to-your-password", stored.get(2).getValue()); + assertEquals("specify the database host", stored.get(0).getValue()); + assertEquals("JDBC host", stored.get(0).getDescription()); + assertEquals("DB_PASSWORD", stored.get(1).getName()); + assertEquals("change-to-your-password", stored.get(1).getValue()); + assertEquals("EMPTY_DEFAULT", stored.get(2).getName()); + assertEquals("", stored.get(2).getValue()); + assertEquals("", stored.get(2).getDescription()); assertTrue( EmbeddedEnvironmentMaterializer.materialize(null, null, "warehouse", tempRoot.toString()) @@ -251,6 +317,34 @@ void materializeKeepsAnExistingConfigurationFile() throws Exception { assertFalse(Files.readString(Path.of(path)).contains("change-to-your-password")); } + @Test + void materializeResolvesTheFolderAndKeepsTheVariableInTheReference() throws Exception { + Path config = tempRoot.resolve("config"); + IVariables variables = new Variables(); + variables.setVariable("HOP_CONFIG_FOLDER", config.toString()); + Project project = new Project(); + project.getEmbeddedEnvironments().add(sampleEnvironment("retail-example-docker-postgres")); + + MaterializeResult result = + EmbeddedEnvironmentMaterializer.materialize( + variables, + new ProjectsConfig(), + project, + "retail-example", + "${HOP_CONFIG_FOLDER}/environments/retail-example/"); + + assertEquals(1, result.getCreated().size()); + String stored = result.getCreated().get(0).getConfigurationFiles().get(0); + assertEquals( + "${HOP_CONFIG_FOLDER}/environments/retail-example/retail-example-docker-postgres.json", + stored); + assertTrue(fileExists(variables.resolve(stored))); + assertFalse( + fileExists( + System.getProperty("user.dir") + + "/${HOP_CONFIG_FOLDER}/environments/retail-example/retail-example-docker-postgres.json")); + } + @Test void materializeRequiresAFolder() { Project project = new Project(); @@ -351,13 +445,15 @@ private EmbeddedEnvironment sampleEnvironment(String name) { .getVariables() .add(new EmbeddedEnvironmentVariable("LOG_LEVEL", "FromDefinition", "Hop log level")); environment - .getMandatoryVariables() - .add(new EmbeddedEnvironmentVariable("DB_HOST", "specify the database host", "JDBC host")); + .getVariables() + .add( + new EmbeddedEnvironmentVariable( + "DB_HOST", "specify the database host", "JDBC host", true)); environment - .getSecretVariables() + .getVariables() .add( new EmbeddedEnvironmentVariable( - "DB_PASSWORD", "change-to-your-password", "JDBC password")); + "DB_PASSWORD", "change-to-your-password", "JDBC password", true, true)); return environment; } diff --git a/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentTest.java b/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentTest.java index 00f2eeb7356..617135f8106 100644 --- a/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentTest.java +++ b/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EmbeddedEnvironmentTest.java @@ -37,7 +37,7 @@ void normalizeDropsBlankNamesAndTrims() { .getVariables() .add(new EmbeddedEnvironmentVariable(" LOG_LEVEL ", " Basic ", " level ")); environment.getVariables().add(new EmbeddedEnvironmentVariable(" ", "ignored", "ignored")); - environment.getMandatoryVariables().add(null); + environment.getVariables().add(null); EmbeddedEnvironmentValidator.normalize(environment); @@ -47,29 +47,28 @@ void normalizeDropsBlankNamesAndTrims() { assertEquals("LOG_LEVEL", environment.getVariables().get(0).getName()); assertEquals("Basic", environment.getVariables().get(0).getDefaultValue()); assertEquals("level", environment.getVariables().get(0).getDescription()); - assertTrue(environment.getMandatoryVariables().isEmpty()); + assertFalse(environment.getVariables().get(0).isMandatory()); + assertFalse(environment.getVariables().get(0).isSecret()); } @Test - void duplicateVariableNameSpansTheThreeLists() { + void legacySecretWithTheSameNameIsADuplicate() { EmbeddedEnvironment environment = environmentNamed("dev"); environment.getVariables().add(new EmbeddedEnvironmentVariable("DB_HOST", "localhost", null)); - environment - .getSecretVariables() - .add(new EmbeddedEnvironmentVariable(" DB_HOST ", "change-to-your-password", null)); + environment.setSecretVariables( + List.of(new EmbeddedEnvironmentVariable(" DB_HOST ", "change-to-your-password", null))); + + EmbeddedEnvironmentValidator.normalize(environment); assertEquals("DB_HOST", EmbeddedEnvironmentValidator.duplicateVariableName(environment)); + assertTrue(environment.getVariables().get(1).isSecret()); } @Test void duplicateVariableNameInsideOneList() { EmbeddedEnvironment environment = environmentNamed("dev"); - environment - .getMandatoryVariables() - .add(new EmbeddedEnvironmentVariable("DB_PORT", "5432", null)); - environment - .getMandatoryVariables() - .add(new EmbeddedEnvironmentVariable("DB_PORT", "5433", null)); + environment.getVariables().add(new EmbeddedEnvironmentVariable("DB_PORT", "5432", null, true)); + environment.getVariables().add(new EmbeddedEnvironmentVariable("DB_PORT", "5433", null, true)); assertEquals("DB_PORT", EmbeddedEnvironmentValidator.duplicateVariableName(environment)); } @@ -79,14 +78,17 @@ void distinctVariableNamesAreAccepted() { EmbeddedEnvironment environment = environmentNamed("dev"); environment.getVariables().add(new EmbeddedEnvironmentVariable("LOG_LEVEL", "Basic", null)); environment - .getMandatoryVariables() - .add(new EmbeddedEnvironmentVariable("DB_HOST", "specify the database host", null)); + .getVariables() + .add(new EmbeddedEnvironmentVariable("DB_HOST", "specify the database host", null, true)); environment - .getSecretVariables() - .add(new EmbeddedEnvironmentVariable("DB_PASSWORD", "change-to-your-password", null)); + .getVariables() + .add( + new EmbeddedEnvironmentVariable( + "DB_PASSWORD", "change-to-your-password", null, true, true)); assertNull(EmbeddedEnvironmentValidator.duplicateVariableName(environment)); assertFalse(EmbeddedEnvironmentValidator.missingName(environment)); + assertTrue(environment.getVariables().get(2).isSecret()); } @Test @@ -119,14 +121,18 @@ void nameTakenMatchesTrimmedNames() { @Test void copyDoesNotShareVariableLists() { EmbeddedEnvironment environment = environmentNamed("dev"); - environment - .getSecretVariables() - .add(new EmbeddedEnvironmentVariable("TOKEN", "change-to-your-token", "API token")); + environment.setSecretVariables( + List.of( + new EmbeddedEnvironmentVariable("TOKEN", "change-to-your-token", "API token", true))); EmbeddedEnvironment copy = environment.copy(); - copy.getSecretVariables().get(0).setDefaultValue("other"); + copy.getVariables().get(0).setDefaultValue("other"); + copy.getVariables().get(0).setMandatory(false); + copy.getVariables().get(0).setSecret(false); - assertEquals("change-to-your-token", environment.getSecretVariables().get(0).getDefaultValue()); + assertEquals("change-to-your-token", environment.getVariables().get(0).getDefaultValue()); + assertTrue(environment.getVariables().get(0).isMandatory()); + assertTrue(environment.getVariables().get(0).isSecret()); } private static EmbeddedEnvironment environmentNamed(String name) { diff --git a/plugins/misc/projects/src/test/java/org/apache/hop/projects/project/ParentProjectFolderSynchronizerTest.java b/plugins/misc/projects/src/test/java/org/apache/hop/projects/project/ParentProjectFolderSynchronizerTest.java index b61699f0ec6..0e40ce4ddb1 100644 --- a/plugins/misc/projects/src/test/java/org/apache/hop/projects/project/ParentProjectFolderSynchronizerTest.java +++ b/plugins/misc/projects/src/test/java/org/apache/hop/projects/project/ParentProjectFolderSynchronizerTest.java @@ -23,7 +23,9 @@ import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; import org.apache.hop.core.logging.HopLogStore; +import org.apache.hop.core.logging.HopLoggingEvent; import org.apache.hop.core.logging.LogChannel; import org.apache.hop.core.variables.IVariables; import org.apache.hop.core.variables.Variables; @@ -182,11 +184,15 @@ void pathTraversalIsRejected() throws Exception { Project project = projectWith(mapping("../escape", false, true, true, null)); + int from = HopLogStore.getLastBufferLineNr(); ParentProjectFolderSynchronizer.synchronize( LogChannel.GENERAL, project, childConfig(child), variables(parent, child)); assertEquals("untouched", Files.readString(escape.resolve("marker.txt"))); assertFalse(Files.exists(escape.resolve("safe.txt"))); + String refused = messagesSince(from); + assertTrue(refused.contains("Refused to copy parent folder '../escape'")); + assertFalse(refused.contains("at org.apache")); } @Test @@ -199,10 +205,16 @@ void invalidExclusionRegexSkipsMapping() throws Exception { Project project = projectWith(mapping("templates", false, true, true, "*.tmp")); + int from = HopLogStore.getLastBufferLineNr(); ParentProjectFolderSynchronizer.synchronize( LogChannel.GENERAL, project, childConfig(child), variables(parent, child)); assertFalse(Files.exists(child.resolve("templates/core.hpl"))); + String ignored = messagesSince(from); + assertTrue(ignored.contains("Ignored invalid exclusion regular expression '*.tmp'")); + assertTrue(ignored.contains("That folder was not copied.")); + assertFalse(ignored.contains("near index")); + assertFalse(ignored.contains("\n")); } @Test @@ -250,6 +262,23 @@ void matchesExclusionUsesBasenameAndRelativePath() { assertFalse(ParentProjectFolderSynchronizer.matchesExclusion(pattern, "keep.hpl", "keep.hpl")); } + private static String messagesSince(int from) { + List events = + HopLogStore.getLogBufferFromTo( + List.of(LogChannel.GENERAL.getLogChannelId()), + true, + from, + HopLogStore.getLastBufferLineNr()); + StringBuilder text = new StringBuilder(); + for (HopLoggingEvent event : events) { + if (text.length() > 0) { + text.append('\n'); + } + text.append(event.getMessage()); + } + return text.toString(); + } + private static Project projectWith(ParentProjectFolder mapping) { Project project = new Project(); project.setParentProjectName("parent"); diff --git a/plugins/misc/projects/src/test/java/org/apache/hop/projects/project/ProjectReferencesTest.java b/plugins/misc/projects/src/test/java/org/apache/hop/projects/project/ProjectReferencesTest.java index 86f615f69fe..7ef4924bf1b 100644 --- a/plugins/misc/projects/src/test/java/org/apache/hop/projects/project/ProjectReferencesTest.java +++ b/plugins/misc/projects/src/test/java/org/apache/hop/projects/project/ProjectReferencesTest.java @@ -18,6 +18,7 @@ package org.apache.hop.projects.project; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -32,6 +33,7 @@ import java.util.List; import org.apache.hop.core.exception.HopException; import org.apache.hop.core.logging.HopLogStore; +import org.apache.hop.core.logging.HopLoggingEvent; import org.apache.hop.core.logging.LogChannel; import org.apache.hop.core.variables.Variables; import org.apache.hop.projects.config.ProjectsConfig; @@ -126,10 +128,16 @@ void parentProjectReferencesSkipProjectsWhichCantBeLoaded() throws Exception { "{ not json", StandardCharsets.UTF_8); + int from = HopLogStore.getLastBufferLineNr(); List references = ProjectsUtil.getParentProjectReferences("ref-parent", new Variables(), LogChannel.GENERAL); assertEquals(List.of("ref-child"), references); + String skipped = messagesSince(from); + assertTrue(skipped.contains("Caught an error loading project 'ref-broken'")); + assertTrue(skipped.contains("That project was skipped.")); + assertFalse(skipped.contains("at org.apache")); + assertFalse(skipped.contains("\n")); } @Test @@ -298,8 +306,13 @@ void failingChildSaveRestoresEveryProject() throws Exception { assumeTrue(secondHome.setWritable(false) && !secondHome.canWrite()); try { + int from = HopLogStore.getLastBufferLineNr(); HopException exception = assertThrows(HopException.class, () -> rename(parentConfig, "ref-parent", "ref-renamed")); + String caught = messagesSince(from); + assertTrue(caught.contains("Caught an error restoring parent project 'ref-parent'")); + assertTrue(caught.contains("of project 'ref-child-b'")); + assertFalse(caught.contains("at org.apache")); assertTrue( exception @@ -340,6 +353,23 @@ void commandLineDeleteRefusesParentProject() throws Exception { .contains("'ref-parent' can't be deleted, it is the parent project of: ref-child")); } + private static String messagesSince(int from) { + List events = + HopLogStore.getLogBufferFromTo( + List.of(LogChannel.GENERAL.getLogChannelId()), + true, + from, + HopLogStore.getLastBufferLineNr()); + StringBuilder text = new StringBuilder(); + for (HopLoggingEvent event : events) { + if (text.length() > 0) { + text.append('\n'); + } + text.append(event.getMessage()); + } + return text.toString(); + } + /** Rename the registered instance in place, the way the project dialogs do. */ private List rename(ProjectConfig projectConfig, String currentName, String newName) throws HopException { diff --git a/plugins/misc/projects/src/test/java/org/apache/hop/projects/project/ProjectTest.java b/plugins/misc/projects/src/test/java/org/apache/hop/projects/project/ProjectTest.java index 6655157c502..8215d4a3c26 100644 --- a/plugins/misc/projects/src/test/java/org/apache/hop/projects/project/ProjectTest.java +++ b/plugins/misc/projects/src/test/java/org/apache/hop/projects/project/ProjectTest.java @@ -119,21 +119,25 @@ public void testEmbeddedEnvironmentsRoundTrip() throws Exception { .getVariables() .add(new EmbeddedEnvironmentVariable("LOG_LEVEL", "Basic", "Hop log level")); environment - .getMandatoryVariables() + .getVariables() .add( - new EmbeddedEnvironmentVariable("DB_HOST", "specify the database host", "JDBC host")); + new EmbeddedEnvironmentVariable( + "DB_HOST", "specify the database host", "JDBC host", true)); environment - .getSecretVariables() + .getVariables() .add( new EmbeddedEnvironmentVariable( - "DB_PASSWORD", "change-to-your-password", "JDBC password")); + "DB_PASSWORD", "change-to-your-password", "JDBC password", true, true)); project.getEmbeddedEnvironments().add(environment); project.saveToFile(); String json = Files.readString(tempFile.toPath()); assertTrue(json.contains("\"defaultValue\"")); - assertTrue(json.contains("\"mandatoryVariables\"")); - assertTrue(json.contains("\"secretVariables\"")); + assertTrue(json.contains("\"mandatory\" : true")); + assertTrue(json.contains("\"secret\" : true")); + assertFalse(json.contains("mandatoryVariables")); + assertFalse(json.contains("secretVariables")); + assertFalse(json.contains("\"secret\" : false")); assertFalse(json.contains("\"configurationFiles\"")); Project read = new Project(tempFile.getAbsolutePath()); @@ -143,17 +147,77 @@ public void testEmbeddedEnvironmentsRoundTrip() throws Exception { assertEquals("Developer workstation", found.getDescription()); assertEquals("LOG_LEVEL", found.getVariables().get(0).getName()); assertEquals("Basic", found.getVariables().get(0).getDefaultValue()); - assertEquals("DB_HOST", found.getMandatoryVariables().get(0).getName()); - assertEquals( - "specify the database host", found.getMandatoryVariables().get(0).getDefaultValue()); - assertEquals("DB_PASSWORD", found.getSecretVariables().get(0).getName()); - assertEquals("change-to-your-password", found.getSecretVariables().get(0).getDefaultValue()); + assertFalse(found.getVariables().get(0).isMandatory()); + assertEquals("DB_HOST", found.getVariables().get(1).getName()); + assertEquals("specify the database host", found.getVariables().get(1).getDefaultValue()); + assertTrue(found.getVariables().get(1).isMandatory()); + assertFalse(found.getVariables().get(0).isSecret()); + assertFalse(found.getVariables().get(1).isSecret()); + assertEquals("DB_PASSWORD", found.getVariables().get(2).getName()); + assertEquals("change-to-your-password", found.getVariables().get(2).getDefaultValue()); + assertTrue(found.getVariables().get(2).isMandatory()); + assertTrue(found.getVariables().get(2).isSecret()); assertNull(read.findEmbeddedEnvironment("Dev")); } finally { tempFile.delete(); } } + @Test + public void testLegacyMandatoryVariablesBecomeFlags() throws Exception { + File tempFile = Files.createTempFile("project-config-legacy-mandatory", ".json").toFile(); + tempFile.deleteOnExit(); + + try { + // Legacy lists are written around variables so property order cannot drop either list. + String legacy = + """ + { + "embeddedEnvironments" : [ { + "name" : "dev", + "secretVariables" : [ { + "name" : "DB_PASSWORD", + "defaultValue" : "change-to-your-password", + "mandatory" : true + } ], + "mandatoryVariables" : [ { + "name" : "DB_HOST", + "defaultValue" : "specify the database host", + "description" : "JDBC host" + } ], + "variables" : [ { + "name" : "LOG_LEVEL", + "defaultValue" : "Basic" + } ] + } ] + } + """; + Files.writeString(tempFile.toPath(), legacy); + + Project read = new Project(tempFile.getAbsolutePath()); + read.readFromFile(); + EmbeddedEnvironment found = read.findEmbeddedEnvironment("dev"); + assertEquals("LOG_LEVEL", found.getVariables().get(0).getName()); + assertFalse(found.getVariables().get(0).isMandatory()); + assertEquals("DB_HOST", found.getVariables().get(1).getName()); + assertEquals("specify the database host", found.getVariables().get(1).getDefaultValue()); + assertTrue(found.getVariables().get(1).isMandatory()); + assertFalse(found.getVariables().get(1).isSecret()); + assertEquals("DB_PASSWORD", found.getVariables().get(2).getName()); + assertTrue(found.getVariables().get(2).isMandatory()); + assertTrue(found.getVariables().get(2).isSecret()); + + read.saveToFile(); + String saved = Files.readString(tempFile.toPath()); + assertFalse(saved.contains("mandatoryVariables")); + assertFalse(saved.contains("secretVariables")); + assertTrue(saved.contains("\"mandatory\" : true")); + assertTrue(saved.contains("\"secret\" : true")); + } finally { + tempFile.delete(); + } + } + @Test public void testMissingEmbeddedEnvironmentsKeyLoadsEmpty() throws Exception { tempRoot = Files.createTempDirectory("hop-project-no-environments");