diff --git a/assemblies/debug/pom.xml b/assemblies/debug/pom.xml index 681a4451f3e..7dfb2f5d2a3 100644 --- a/assemblies/debug/pom.xml +++ b/assemblies/debug/pom.xml @@ -571,6 +571,12 @@ ${project.version} provided + + org.apache.hop + hop-transform-watchfiles + ${project.version} + provided + org.apache.hop hop-transform-writetolog diff --git a/assemblies/plugins/pom.xml b/assemblies/plugins/pom.xml index d1880833cbe..f4ede4de0b7 100644 --- a/assemblies/plugins/pom.xml +++ b/assemblies/plugins/pom.xml @@ -1596,6 +1596,12 @@ ${project.version} zip + + org.apache.hop + hop-transform-watchfiles + ${project.version} + zip + org.apache.hop hop-transform-webserviceavailable diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/watchfiles.svg b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/watchfiles.svg new file mode 100644 index 00000000000..6ef6002720e --- /dev/null +++ b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/watchfiles.svg @@ -0,0 +1 @@ + diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/transforms/watchfiles-general.png b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/watchfiles-general.png new file mode 100644 index 00000000000..4960bdcf851 Binary files /dev/null and b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/watchfiles-general.png differ diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/transforms/watchfiles-maintenance.png b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/watchfiles-maintenance.png new file mode 100644 index 00000000000..99d93ea0f61 Binary files /dev/null and b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/watchfiles-maintenance.png differ diff --git a/docs/hop-user-manual/modules/ROOT/nav.adoc b/docs/hop-user-manual/modules/ROOT/nav.adoc index 2546e207f4d..24875e09bbb 100644 --- a/docs/hop-user-manual/modules/ROOT/nav.adoc +++ b/docs/hop-user-manual/modules/ROOT/nav.adoc @@ -334,6 +334,7 @@ under the License. *** xref:pipeline/transforms/vcardinput.adoc[vCard Input] *** xref:pipeline/transforms/vcardoutput.adoc[vCard Output] *** xref:pipeline/transforms/verticabulkloader.adoc[Vertica Bulk Loader] +*** xref:pipeline/transforms/watchfiles.adoc[Watch Files] *** xref:pipeline/transforms/webservices.adoc[Web services lookup] *** xref:pipeline/transforms/workflow-executor.adoc[Workflow Executor] *** xref:pipeline/transforms/workflow-logging.adoc[Workflow logging] diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/watchfiles.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/watchfiles.adoc new file mode 100644 index 00000000000..830ed3b5d65 --- /dev/null +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/watchfiles.adoc @@ -0,0 +1,295 @@ +//// +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +//// +:documentationPath: /pipeline/transforms/ +:language: en_US +:description: Watch Files continuously emits stable file changes with persistent state and reconciliation. +:page-engines: Hop Engine=yes, Single Threaded=no, Native Spark=no, Beam Spark=no, Beam Flink=no, Beam Dataflow=no + += image:transforms/icons/watchfiles.svg[Watch Files transform Icon, role="image-doc-icon"] Watch Files + +== Description + +Watch Files is a continuously running source transform. It emits a metadata row for a matching file that is created, modified or deleted. +Use exactly one copy with the normal Local Hop Engine, including that engine on Hop Server. +Stop the pipeline to end monitoring, or configure an optional maximum run time in General. + +xref:pipeline/transforms/getfilenames.adoc[Get File Names] lists files in a finite operation. +xref:workflow/actions/waitforfile.adoc[Wait for file] waits for one filename in a workflow and can wait for it to stop growing. +Watch Files maintains a directory's observed state across runs and produces a stream of changes. +Neither existing component is changed. + +[source,text] +---- +Local WatchService hints ----+ + +--> Compare filesystem with observed state +VFS polling snapshots -------+ | + Coalesce by URI + | + Check stability + | + Emit a Hop row + | + Acknowledge observed version + | + Periodic JSON checkpoint +---- + +== Configuration + +The dialog has three tabs: + +* *General*: folder, pattern syntax, filename filters, recursion, event types, file readiness and first-start behavior. Visible examples follow the pattern syntax selected in General. +* *Advanced*: detection strategy, intervals, resource limits, checkpoint location/identity and diagnostic tuning. Stability check details are hidden when readiness checking is disabled; native-only settings are hidden for explicit polling. Native detection shows Reconciliation interval and hides Polling interval; Automatic shows both because its detection method is chosen at startup. The retained polling value still controls retries in native mode; switch to Automatic to edit it. Hover help follows the selected strategy and filename syntax. Hidden values are preserved. Minimum file age still applies when readiness checking is disabled. +* *Maintenance*: View status, Create backup and Reprocess files. Advanced recovery reveals Clear saved history, Restore backup and Update saved-state format. The result panel appears after an operation; the reprocessing filter opens only when choosing Reprocess files. + +General shows *State identifier: Automatic* for the generated UUID, or *State identifier: Saved* for an existing custom ID. Hover over it to inspect the actual saved value. New transforms receive a generated Watch ID once, saved in the pipeline. Existing IDs and state directories are preserved. To edit the value deliberately, select *Advanced -> Edit state identifier*. Turning editing off hides the field and retains its value; it never generates another ID. The editing preference is saved in the pipeline and does not affect checkpoint identity or validation of the directory and filters. +The default state directory for new transforms is `/.hop/watch-files`, persisted as an explicit local path. +Review that path when moving a pipeline to another host, and configure a directory outside the watched tree if watching the user home directory or an ancestor. + +image::transforms/watchfiles-general.png[Watch Files General settings, width="900"] + +All paths, patterns and numeric settings accept Hop variables. +Intervals are integer milliseconds (1 to 2147483647; minimum age also accepts zero). +Maximum run time uses a positive number of minutes or hours, including decimals with a decimal point, or a variable. Blank means continuous monitoring. +Timing begins when row processing starts, after initialization, and uses monotonic elapsed time. +When the duration expires, the source saves acknowledged observations and signals normal end of input. Downstream rows already delivered can finish, and unacknowledged changes can be detected on the next run with the same state identifier. +This is a cooperative limit: an in-progress filesystem operation, checkpoint or downstream handoff can delay completion. It is not a deadline for the entire pipeline or its downstream work. +Expressions match the *entire short filename*, as in Get File Names; exclusion wins. +Empty include/exclude expressions mean include all/exclude none. +Only regular files generate rows; directories are traversed but never emitted. + +[options="header"] +|=== +|Option |Default |Description +|Directory |Required |Directory to monitor. Variables and named VFS connections are supported. +|Edit state identifier |Off |Advanced option that reveals the saved identifier for manual editing. Hiding the field retains the identifier and its history. +|State identifier (Watch ID) |Generated for new transforms |Stable identifier saved in the pipeline, independent of the transform name. Existing IDs are retained; enable Edit state identifier in Advanced to change it. A different ID starts a separate history without overwriting the previous checkpoint. +|Pattern syntax |WILDCARD for newly created transforms; REGEXP for existing XML without the setting |WILDCARD uses `*` for any text and `?` for one character; all other characters are literal. REGEXP uses Java regular expressions. Matching is case-sensitive against the entire short filename. +|Include filename pattern |Empty |Pattern using the selected syntax. Empty includes all files. +|Exclude filename pattern |Empty |Pattern using the selected syntax. Empty excludes nothing. Exclusion takes precedence. +|Include subdirectories |false |Recursively monitor directories without following symbolic links. +|Detection strategy |AUTO |AUTO uses a local WatchService when supported; NATIVE requires it; POLLING always scans. +|Created |true |Emit CREATED events. +|Modified |true |Emit MODIFIED events. +|Deleted |false |Emit DELETED events using the previous file metadata. +|Polling interval (ms) |5000 |Interval between complete VFS polling scans. +|Reconciliation interval (ms) |60000 |Interval between complete native reconciliation scans. +|Native hint capacity |4096 |Maximum paths and keys drained per iteration. Excess hints trigger reconciliation. +|Maximum entries |100000 |Maximum files, pending candidates and registered directories. Exceeding this limit fails safely. +|Wait until stable |true |Require consecutive unchanged size and modification time observations. +|Minimum file age (ms) |2000 |Require this age since last modification, even when stability checks are disabled. +|Stability checks |2 |Number of consecutive unchanged observations after the initial observation. +|Stability interval (ms) |1000 |Minimum time between observations of a pending file. +|State directory |User home .hop/watch-files for new transforms |Local durable state directory outside the watched tree; existing pipeline paths are retained. +|First start without saved state |Watch new changes only |Process existing files (EMIT_EXISTING) or establish a quiet baseline (IGNORE_EXISTING / legacy COMPARE_WITH_STATE). An existing checkpoint is always compared. Opening and saving the dialog preserves the legacy enum value. +|Checkpoint interval (ms) |5000 |Checkpoint acknowledged observations at this interval and on graceful disposal. +|Maximum run time |Empty (continuous) |Optional positive duration, for example 30 minutes or 1.5 hours. At the limit, save state and finish normally. Variables are supported. +|Run time unit |Minutes |Minutes or Hours. Visible only when a duration is entered; hiding the setting retains its value. +|Diagnostics interval (ms) |60000 |Log a snapshot of counters and operation durations; also logged on shutdown. +|Slow operation threshold (ms) |30000 |Warn after a completed scan, metadata probe or checkpoint exceeds this duration. +|Filename pattern (Reprocess files) |Saved replayFilter or .* |One-time selection in the Reprocess files window. Existing saved filters remain the initial choice; selecting a different filter does not rewrite the pipeline configuration. +|=== + +=== Detection strategies + +* *AUTO*: a Commons VFS `LocalFile` with the `file` scheme uses Java WatchService when registration succeeds. + Unsupported native registration falls back to VFS polling. A resource limit failure does not fall back. +* *NATIVE*: requires a local, readable, existing directory and a functioning WatchService; unsupported locations fail initialization. +* *POLLING*: always scans through Commons VFS, including local and mounted filesystems. + +Local watching uses one WatchService and no auxiliary threads. +Existing subdirectories are registered before the initial scan. Newly created or moved directory trees trigger priority reconciliation and registration. +An overflow, invalid WatchKey or excess native hints requests priority reconciliation. +Ordinary file hints read only the affected file's metadata. +The normal Local Hop Engine gives this transform its own execution thread; the Single Threaded engine is excluded. + +Remote schemes use Hop's installed VFS providers; this plugin creates no SFTP, FTP or cloud clients. +Root resolution uses `HopVfs.getFileObject(resolvedPath, variables)`, preserving execution variables, metadata-driven connections and VFS namespaces. +Subsequent operations resolve relative to that same root FileObject. +Named connections are recommended, particularly on Hop Server. Relative paths follow the existing HopVfs working-directory rules; +use an explicit variable such as `${PROJECT_HOME}` for a project-relative path. + +=== Stability and coalescing + +A candidate holds the latest size and last-modified timestamp and the previously acknowledged version. +Repeated operating-system hints for an unchanged version do not restart the stability interval. +A changed size *or* timestamp resets consecutive successful checks to zero. +The configured number of unchanged observations, separated by at least the stability interval, is required before emission. +Minimum age is measured from last modification and applies even if stability checking is disabled. +A fresh metadata read is performed before each emission. +An unacknowledged new file remains CREATED while it grows; an acknowledged file becomes MODIFIED. +Files which disappear before their first emission are removed from the candidate set. + +This is a readiness heuristic, not a writer lock: a writer can pause longer than the configured interval or resume after emission. +For a strict producer protocol, write to an excluded temporary name and rename to a matching final filename. +Files with a future last-modified timestamp wait until they meet minimum age. Keep host clocks synchronized. + +Polling, reconciliation, checkpoints and stability intervals use elapsed monotonic time. +Wall-clock adjustments do not advance these intervals. Minimum age and output dates still use wall time and filesystem timestamps. +The polling/reconciliation interval starts after the previous scan completes, leaving time for readiness checks and emission even when a scan exceeds its interval. + +=== State and restarts + +Keep the generated or existing Watch ID stable, independent of the transform name. It is not regenerated at execution time. +The state directory must be local and outside the watched tree. +One process holds an exclusive OS file lock per Watch ID for the entire run. +Another transform, pipeline or JVM using the same state directory and ID fails instead of racing. + +Each versioned checkpoint contains Watch ID, resolved root, recursion/filter scope and metadata for each acknowledged URI. +Schema 2 additionally records `semantics=OBSERVED`, a checkpoint generation and wall-clock save time. +Schema 1 is read without losing observations; the first subsequent save archives its original artifacts and upgrades it to schema 2. +Unknown future schemas are rejected. +Changing the root, recursion or resolved patterns while reusing an ID fails safely; choose a new ID for a new scope. +For example, to expand the CSV sample to all files, clear both filename filters and use a new Watch ID such as `manual-all-files`. +An empty include filter includes all filenames. With *Pattern syntax = WILDCARD*, `*` also includes all files, `*test*` selects names containing `test`, `*.csv` selects CSV files and `file?.txt` selects a single character after `file`. +With *Pattern syntax = REGEXP*, `.*` includes all files; `*` alone is invalid. Existing pipelines that do not contain a `patternSyntax` setting retain their regular-expression interpretation and checkpoint scope. +Variables are resolved before pattern conversion. Checkpoint scope uses the resolved effective regular expressions, so a syntax change that changes the matched scope requires a new ID or an explicit reset. +The *Reprocess files* pattern always uses a regular expression regardless of filename pattern syntax. +If all files already present must also be emitted, select *EMIT_EXISTING* before the first run with the new ID. +Alternatively, stop the pipeline and use *Maintenance -> Advanced recovery -> Clear saved history* to archive and reset the existing ID explicitly before changing its scope. +The startup error distinguishes a changed Directory from changed filters or recursion; neither mismatch means the saved JSON is corrupt. +Event enablement can be changed without changing scope. Disabled events still advance observed state. + +All initial-scan modes compare against an existing valid checkpoint. +Without a checkpoint: + +* *EMIT_EXISTING*: emit existing matching files as CREATED after readiness checks. +* *IGNORE_EXISTING*: persist a quiet baseline immediately. +* *COMPARE_WITH_STATE*: also persist a quiet baseline immediately. + +Example: baseline A and B; emit C; stop; create D offline; restart. +The initial comparison emits D and retains A, B and C without repeating them. + +Rows are acknowledged in memory only after `putRow` returns successfully and the pipeline remains running. +Checkpoints run periodically and during disposal. Pending unstable candidates are never checkpointed as emitted. +The checkpoint describes the source's *observed version handed to Hop*, not completion by downstream transforms. +An abrupt crash before the next checkpoint can repeat rows. +A downstream failure after a successful checkpoint can require explicit replay; there is no downstream acknowledgement transaction. +This is recoverable filesystem-state detection with possible redelivery, not end-to-end exactly-once or a guaranteed durable event log. + +=== Checkpoint durability and recovery + +The checkpoint is written to a temporary sibling file, fully closed and forced to storage, then replaced using `ATOMIC_MOVE`. +NIO is intentionally used for this local persistence operation: a generic VFS copy/delete move does not provide checkpoint atomicity. + +If atomic replacement is unsupported, the previous checkpoint is copied to a forced `.bak` file and a forced `.commit` intent is created before replacement. +A completed fallback removes the intent. +An interrupted fallback leaves the backup and intent; initialization and further saves refuse to proceed. +Use *Restore backup* under Advanced recovery in the Maintenance tab to validate a `.json.bak` or archived `.json`, archive current artifacts and restore it before clearing the intent. +If no backup exists (first checkpoint), explicitly decide whether to start over with a new ID. +The fallback protects the last known-good checkpoint and fails closed; it cannot promise atomic replacement on an unsupported filesystem. + +Corrupt JSON, unknown/missing versions, invalid entries, an incompatible scope or an interrupted commit never become an empty baseline. +Initialization fails with an actionable message and leaves the artifacts intact. +Restore a verified checkpoint or use a new Watch ID intentionally. +A stale `.tmp` file from an interrupted atomic write is ignored while the intact main checkpoint is loaded. + +Directory forcing is best effort because Windows and some filesystems cannot force directory handles. +Local OS locks and fsync semantics vary on network-mounted drives; use a reliable durable local state volume. +Choose a durable state directory explicitly. Temporary directories may be cleaned by the OS and are unsuitable for production checkpoints. +Checkpoint metadata contains filenames and URIs; protect it with filesystem permissions and use named VFS connections rather than embedding credentials in URLs. + +=== Operator recovery and diagnostics + +image::transforms/watchfiles-maintenance.png[Watch Files Maintenance actions, width="900"] + +The *Maintenance* tab shows three everyday actions: *View status*, *Create backup* and *Reprocess files*. Select *Advanced recovery* to reveal *Clear saved history*, *Restore backup* and *Update saved-state format*. The results area is hidden until an action produces a result or error. Each action has hover help. +Actions resolve the state directory and Watch ID from the current form. They operate immediately, independently of OK/Cancel; cancelling the configuration dialog does not undo an explicit recovery action. +Mutations require the watcher to be stopped and acquire the same OS lock as a running source. +Each destructive action asks for confirmation and first archives the reserved artifacts for that Watch ID in `.history/-`. +Backups are retained until an operator removes them; monitor disk usage and apply your own retention policy. + +* *View status* shows a stopped checkpoint's schema, generation, root and observed count. In the same JVM, it can show the running source's immutable diagnostics snapshot. A source running in another process retains its lock and must be inspected through that process's logs. +* *Create backup* archives the current artifacts, including corrupt files for investigation, without changing the checkpoint. +* *Reprocess files* opens a separate window to choose a one-time filename regular expression. The input uses grouped annotated metadata widgets, validates the expression and leaves history unchanged when cancelled. Confirming the subsequent state action removes only observations matching that selection. The saved pipeline replayFilter is retained as the initial choice and is not changed by the one-time selection. On the next start, matching files still present are eligible for CREATED after readiness checks. Enable Created before replay. Already deleted content cannot be recovered, and downstream duplicates are possible. +* *Clear saved history* backs up and removes only this Watch ID's checkpoint artifacts. The next start follows First start without saved state; choose Process existing files deliberately if all existing matching files must be re-emitted. +* *Restore backup* validates the selected checkpoint before replacing current state. It preserves the previous artifacts and records durable commit intent during restoration. Scope compatibility is enforced when the watcher next starts. +* *Update saved-state format* explicitly rewrites valid schema 1 state as schema 2 without changing its observed entries, preserving a backup. + +Periodic diagnostics include observed and pending counts, successful scans/checkpoints, retries, native overflows, emitted/suppressed events, last successful scan time and most recent scan/checkpoint duration. +Slow-operation warnings identify completed scans, stats and checkpoints; they do not cancel blocked VFS calls. Configure the provider's connection/session timeouts as well. + +=== Reconciliation, errors and limits + +Complete scans compare actual metadata with acknowledged state and recover missed create/modify/delete changes. +Failed, incomplete or cancelled scans never infer deletions; the last valid state is retained and reconciliation retries at the polling interval. +Root disappearance or permission failure during execution is treated as an unavailable scan, rather than mass deletion. +Stability probe failures retain candidates for retry. +Invalid startup roots, regexes, state directories and checkpoints fail initialization. +Checkpoint write failures fail the transform. +Configured file/directory/candidate limits fail safely and require a larger limit; they are not transient retries. + +Native hints are capped by Native hint capacity. Excess hints request reconciliation rather than accumulating without bound. +Acknowledged files and pending candidates are bounded by Maximum entries, as are visited/registered directories. +Hop's bounded rowsets provide downstream backpressure. No unbounded output-event queue or scheduler is created. +A slow downstream can delay watching, scans and checkpoints; WatchService overflow then requests a scan when execution resumes. +A VFS provider may materialize a full directory listing internally; configure provider connection/read timeouts and size the process for the monitored tree. +Closing the WatchService wakes the native poll, and polling waits are notified on Stop. In-flight remote calls depend on the provider's timeout support. +Configure finite connection and session/read timeouts on named VFS connections. Providers such as SFTP may retry a timed-out read, so stop latency can exceed a single configured timeout. + +Symbolic links are skipped and recursive native traversal does not follow links. +Providers must report symbolic links accurately; visited-directory identities also prevent simple loops. +Windows drive paths, file URIs and UNC paths are resolved through VFS; native paths use NIO Path normalization rather than string prefix matching. +Local file URIs are normalized using UTF-8 decoding. Literal percent signs in native paths are escaped for VFS resolution, and filters use decoded base filenames. +Use POLLING for network mounts with unreliable native events. + +Reconciliation recovers *persisting state differences*, not every transient operation. +A file created and removed entirely between observations cannot be reconstructed. +Changes which restore the previous size and timestamp cannot be detected without a content/identity strategy. +Renames are modeled as DELETED old URI and CREATED new URI. +No content hashes or content reads are performed. + +== Output fields + +[options="header"] +|=== +|Field |Hop type |Meaning +|filename |String |Local filename or VFS URI, usable by existing filename-from-field readers. +|short_filename |String |Base filename. +|path |String |Parent directory URI. +|uri |String |Normalized VFS file URI used as identity. +|event_type |String |CREATED, MODIFIED or DELETED. +|size |Integer |Size in bytes (Long). +|last_modified |Date |File modification time. +|detected_at |Date |Time the emitted version was detected as ready; deletion uses the reconciliation time. +|is_directory |Boolean |Always false; this source emits regular files. +|scheme |String |VFS scheme. +|previous_size |Integer |Previously acknowledged size; null for CREATED. +|previous_last_modified |Date |Previously acknowledged modification time; null for CREATED. +|watch_id |String |Resolved Watch ID. +|=== + +DELETED rows carry the previous file's metadata because the file no longer exists. +Filter out DELETED before sending filenames to a reader. +For local filenames containing a literal percent sign, `filename` is a file URI so existing VFS readers can resolve the escape safely. + +== Example + +The bundled `watch-files/watch-files.hpl` sample monitors `${PROJECT_HOME}/watch-files/input` and writes filename, event type, size and modification time to the log. +Create or modify a CSV in that directory, wait for stability, then stop the pipeline. +Restart it after adding a CSV while stopped to observe recovery. +Its sibling `state` directory persists checkpoints. + +For ingestion, connect Watch Files to Filter Rows (event_type != DELETED), then Text File Input or JSON Input configured to take filenames from the `filename` field. +Readers remain responsible for content parsing and their usual error handling. + +== Scope + +The initial implementation supports the normal Local Hop Engine only. +It has no JDBC state backend, distributed state, HA coordination, leader election, Beam/Spark/Flink support or mandatory checksum. +Windows and Ubuntu/OpenSSH SFTP behavior, Linux native overflow and SWT GUI interaction have integration coverage. Other remote providers, macOS/UNC behavior, large-scale capacity and power-loss durability still need validation in the target deployment. diff --git a/plugins/transforms/pom.xml b/plugins/transforms/pom.xml index 37abb86be84..810fa12f094 100644 --- a/plugins/transforms/pom.xml +++ b/plugins/transforms/pom.xml @@ -172,6 +172,7 @@ validator valuemapper vcard + watchfiles webserviceavailable webservices workflowexecutor diff --git a/plugins/transforms/watchfiles/TESTING.md b/plugins/transforms/watchfiles/TESTING.md new file mode 100644 index 00000000000..78664175dde --- /dev/null +++ b/plugins/transforms/watchfiles/TESTING.md @@ -0,0 +1,59 @@ + + +# Testing Watch Files + +Use Java 21 and the repository Maven wrapper: + +```sh +./mvnw -B -pl plugins/transforms/watchfiles -Pskip-uitest clean install apache-rat:check spotless:check +tools/with-isolated-display.sh ./mvnw -B -pl plugins/transforms/watchfiles -Puitest -Dwatchfiles.gui.capture=true test +``` + +The first command runs portable headless tests and Linux filesystem tests when +supported. The second runs SWT GUI tests on a separate Linux display and optionally +captures screenshots under the module's `target` directory. + +Coverage includes stable changes, recursive registration, overflow reconciliation, +exclusive state ownership, corrupt or incompatible checkpoints, atomic replacement +and interrupted fallback commits, separate-JVM termination, monotonic scheduling, +VFS paths (including Unicode, literal percent signs, Windows drive/UNC roots and +unusual valid Unix names), invalid native-key cleanup after capped drains, +directory recreation, source restarts, timeout completion and downstream draining, plus GUI +save/cancel, conditional fields and recovery actions. + +Environment-specific tests are opt-in: + +* SFTP: set `WATCHFILES_SFTP_HOST`, `WATCHFILES_SFTP_USER`, + `WATCHFILES_SFTP_PASSWORD`, `WATCHFILES_SFTP_ROOT`, and optionally + `WATCHFILES_SFTP_PORT`. Use a disposable test root; credentials belong in the + process environment, not checked-in configuration. +* Disk full: set `WATCHFILES_FULL_FILESYSTEM` to a disposable, already-full local + filesystem for the ENOSPC test. +* Sustained load: use `-Dtest=WatchFilesLoadTest -Dwatchfiles.load.seconds=86400`. + The native/polling test uses concurrent writers, bounded rowsets, a slow sink + and restarts. Its JSONL report defaults to `target/watchfiles-load.jsonl`. + +The original implementation passed a dedicated Linux soak lasting 86,444 seconds, +with 94 cycles, 187,000 consumed events, zero duplicate and zero unexpected rows. +Later filename-pattern, GUI and timeout changes have separate regression checks; +this historical soak does not constitute a new 24-hour run of those changes. + +State records observations handed to Hop, independently of downstream completion. +Tests do not establish end-to-end exactly-once delivery, power-loss durability, +macOS support or coverage of every remote VFS provider. Full reactor CI remains +the merge gate. diff --git a/plugins/transforms/watchfiles/pom.xml b/plugins/transforms/watchfiles/pom.xml new file mode 100644 index 00000000000..ce138872b8a --- /dev/null +++ b/plugins/transforms/watchfiles/pom.xml @@ -0,0 +1,49 @@ + + + + 4.0.0 + + org.apache.hop + hop-plugins-transforms + 2.20.0-SNAPSHOT + + hop-transform-watchfiles + jar + Hop Plugins Transforms Watch Files + + + + org.apache.hop + hop-libs + ${project.version} + pom + import + + + + + + com.github.mwiede + jsch + test + + + org.apache.hop + hop-tech-sftp + ${project.version} + test + + + org.apache.hop + hop-transform-writetolog + ${project.version} + test + + + org.junit.platform + junit-platform-launcher + ${junit.version} + test + + + diff --git a/plugins/transforms/watchfiles/src/assembly/assembly.xml b/plugins/transforms/watchfiles/src/assembly/assembly.xml new file mode 100644 index 00000000000..bf562cde866 --- /dev/null +++ b/plugins/transforms/watchfiles/src/assembly/assembly.xml @@ -0,0 +1,46 @@ + + + + hop-transform-watchfiles + + zip + + . + + + ${project.basedir}/src/main/resources/version.xml + ${hop.plugin.libdir} + true + + + + + + ${project.basedir}/src/main/samples + config/projects/samples/ + + + + + + ${maven.multiModuleProjectDirectory}/assemblies/shared/hop-plugin-libs.xml + + diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/DetectionStrategy.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/DetectionStrategy.java new file mode 100644 index 00000000000..6906f4496e2 --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/DetectionStrategy.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +public enum DetectionStrategy { + AUTO, + NATIVE, + POLLING +} diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FileChangeEvent.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FileChangeEvent.java new file mode 100644 index 00000000000..b9056909b47 --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FileChangeEvent.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import lombok.Value; + +@Value +public class FileChangeEvent { + FileEventType type; + FileState current; + FileState previous; + long detectedAt; + + public FileState file() { + return current == null ? previous : current; + } +} diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FileEventType.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FileEventType.java new file mode 100644 index 00000000000..630700d7d14 --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FileEventType.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +public enum FileEventType { + CREATED, + MODIFIED, + DELETED +} diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FileReconciler.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FileReconciler.java new file mode 100644 index 00000000000..72607a925d9 --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FileReconciler.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import java.util.Map; +import java.util.function.Consumer; + +/** Only complete successful snapshots may be passed here: a failed listing is not a deletion. */ +public class FileReconciler { + public void compare( + Map observed, + Map current, + long now, + Consumer changes) { + for (FileState file : current.values()) { + FileState previous = observed.get(file.getUri()); + if (previous == null) { + changes.accept(new FileChangeEvent(FileEventType.CREATED, file, null, now)); + } else if (!file.sameVersion(previous)) { + changes.accept(new FileChangeEvent(FileEventType.MODIFIED, file, previous, now)); + } + } + for (FileState file : observed.values()) { + if (!current.containsKey(file.getUri())) { + changes.accept(new FileChangeEvent(FileEventType.DELETED, null, file, now)); + } + } + } +} diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FileStabilityTracker.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FileStabilityTracker.java new file mode 100644 index 00000000000..87f3606ea2b --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FileStabilityTracker.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** One candidate per URI. Repeated hints do not restart an unchanged candidate's quiet interval. */ +public class FileStabilityTracker { + private final Map candidates = new LinkedHashMap<>(); + private final boolean waitUntilStable; + private final long minimumAge; + private final int checks; + private final long interval; + private final int capacity; + + public FileStabilityTracker( + boolean waitUntilStable, long minimumAge, int checks, long interval, int capacity) { + this.waitUntilStable = waitUntilStable; + this.minimumAge = minimumAge; + this.checks = checks; + this.interval = interval; + this.capacity = capacity; + } + + public void observe(FileChangeEvent event, long now) { + String uri = event.file().getUri(); + if (event.getType() == FileEventType.DELETED) { + candidates.remove(uri); + return; + } + Candidate existing = candidates.get(uri); + if (existing != null && event.getCurrent().sameVersion(existing.event.getCurrent())) { + return; + } + if (existing == null && candidates.size() >= capacity) { + throw new IllegalStateException("Pending file limit exceeded. Increase maximum entries."); + } + // Previous always comes from the committed observed state, not another native hint. + candidates.put(uri, new Candidate(event, now)); + } + + public boolean due(String uri, long now) { + return due(uri, now, now); + } + + public boolean due(String uri, long now, long wallTime) { + Candidate candidate = candidates.get(uri); + return candidate != null + && wallTime - candidate.event.getCurrent().getLastModified() >= minimumAge + && (!waitUntilStable || now - candidate.checkedAt >= interval); + } + + public FileChangeEvent check(String uri, FileState current, long now) { + return check(uri, current, now, now); + } + + public FileChangeEvent check(String uri, FileState current, long now, long wallTime) { + Candidate candidate = candidates.get(uri); + if (candidate == null) { + return null; + } + if (current == null) { + candidates.remove(uri); + return null; + } + if (!current.sameVersion(candidate.event.getCurrent())) { + candidate.event = + new FileChangeEvent( + candidate.event.getType(), current, candidate.event.getPrevious(), wallTime); + candidate.checkedAt = now; + candidate.equalChecks = 0; + return null; + } + if (now - candidate.checkedAt >= interval) { + candidate.checkedAt = now; + candidate.equalChecks++; + } + if (wallTime - current.getLastModified() < minimumAge + || (waitUntilStable && candidate.equalChecks < checks)) { + return null; + } + return new FileChangeEvent( + candidate.event.getType(), current, candidate.event.getPrevious(), wallTime); + } + + public String[] uris() { + return candidates.keySet().toArray(String[]::new); + } + + public int size() { + return candidates.size(); + } + + /** Rotate candidates in O(1), without copying the whole tree for every emitted row. */ + public String nextUri() { + if (candidates.isEmpty()) { + return null; + } + String uri = candidates.keySet().iterator().next(); + Candidate candidate = candidates.remove(uri); + candidates.put(uri, candidate); + return uri; + } + + public void remove(String uri) { + candidates.remove(uri); + } + + private static class Candidate { + private FileChangeEvent event; + private long checkedAt; + private int equalChecks; + + private Candidate(FileChangeEvent event, long now) { + this.event = event; + this.checkedAt = now; + } + } +} diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FileState.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FileState.java new file mode 100644 index 00000000000..91449c48413 --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FileState.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * Metadata only; content is never opened or hashed. Treat instances as immutable after creation. + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class FileState { + private String uri; + private String filename; + private String shortFilename; + private String path; + private String scheme; + private long size; + private long lastModified; + + public boolean sameVersion(FileState other) { + return other != null && size == other.size && lastModified == other.lastModified; + } +} diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FileStateStore.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FileStateStore.java new file mode 100644 index 00000000000..9f650d7c871 --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FileStateStore.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import java.io.IOException; +import java.util.Map; + +public interface FileStateStore extends AutoCloseable { + /** Null means no checkpoint; invalid or ambiguous checkpoints must throw. */ + Map load() throws IOException; + + void save(Map files) throws IOException; + + @Override + void close() throws IOException; +} diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FileWatcher.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FileWatcher.java new file mode 100644 index 00000000000..d523abefa0f --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FileWatcher.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Set; +import lombok.Value; + +/** Hints are bounded. A reconciliation flag replaces excess hints without losing final state. */ +public interface FileWatcher extends AutoCloseable { + WatchBatch poll(long timeoutMillis) throws IOException, InterruptedException; + + default void reconcileDirectories() throws IOException {} + + @Override + void close() throws IOException; + + @Value + class WatchBatch { + Set paths; + boolean reconciliationRequired; + long overflowCount; + } +} diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FilenamePatternSyntax.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FilenamePatternSyntax.java new file mode 100644 index 00000000000..409c388cadb --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/FilenamePatternSyntax.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import java.util.regex.Pattern; + +/** Explicit filename syntax avoids guessing whether an existing expression is a wildcard. */ +public enum FilenamePatternSyntax { + REGEXP, + WILDCARD; + + public String toRegex(String value) { + if (value == null || value.isEmpty()) { + return ""; + } + if (this == REGEXP) { + return value; + } + StringBuilder regex = new StringBuilder("(?s)"); + value + .codePoints() + .forEach( + character -> { + switch (character) { + case '*' -> regex.append(".*"); + case '?' -> regex.append('.'); + default -> regex.append(Pattern.quote(new String(Character.toChars(character)))); + } + }); + return regex.toString(); + } +} diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/InitialScan.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/InitialScan.java new file mode 100644 index 00000000000..40d55577924 --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/InitialScan.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +/** With an existing checkpoint, every mode compares against it. */ +public enum InitialScan { + EMIT_EXISTING, + IGNORE_EXISTING, + COMPARE_WITH_STATE +} diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/JsonFileStateStore.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/JsonFileStateStore.java new file mode 100644 index 00000000000..1b375967bee --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/JsonFileStateStore.java @@ -0,0 +1,318 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.io.InterruptedIOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; +import java.nio.file.AccessDeniedException; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.HashMap; +import java.util.Map; +import lombok.Getter; +import lombok.Setter; + +/** + * Local checkpoints deliberately use NIO: VFS moveFile can copy/delete across mounts, which is + * unsuitable for a durable checkpoint. The writer holds an OS lock for its entire lifetime. + */ +public class JsonFileStateStore implements FileStateStore { + private final ObjectMapper mapper = + new ObjectMapper().enable(DeserializationFeature.FAIL_ON_TRAILING_TOKENS); + private final Path state; + private final Path temporary; + private final Path backup; + private final Path journal; + private final String watchId; + private String root; + private String scope; + private final int maximumEntries; + private final FileChannel lockChannel; + private final FileLock lock; + private boolean loaded; + private boolean recoveryRequired; + private int loadedVersion; + private long generation; + + public JsonFileStateStore( + Path directory, String watchId, String root, String scope, int maximumEntries) + throws IOException { + if (!watchId.matches("[A-Za-z0-9][A-Za-z0-9_-]{0,127}")) { + throw new IOException("Watch ID must contain 1-128 letters, digits, underscores or hyphens."); + } + this.watchId = watchId; + this.root = root; + this.scope = scope; + this.maximumEntries = maximumEntries; + Files.createDirectories(directory); + state = directory.resolve(watchId + ".json"); + temporary = directory.resolve(watchId + ".json.tmp"); + backup = directory.resolve(watchId + ".json.bak"); + journal = directory.resolve(watchId + ".json.commit"); + lockChannel = + FileChannel.open( + directory.resolve(watchId + ".lock"), + StandardOpenOption.CREATE, + StandardOpenOption.WRITE); + FileLock acquired; + try { + acquired = lockChannel.tryLock(); + if (acquired == null) { + throw new IOException("Watch ID is already in use: " + watchId); + } + } catch (IOException | OverlappingFileLockException e) { + lockChannel.close(); + throw new IOException("Cannot acquire exclusive state lock for Watch ID " + watchId, e); + } + lock = acquired; + } + + @Override + public Map load() throws IOException { + if (Files.exists(journal)) { + recoveryRequired = true; + throw new IOException( + "Interrupted non-atomic checkpoint for " + + watchId + + ". Preserve all state files and restore the .bak checkpoint before removing .commit."); + } + if (!Files.exists(state)) { + loaded = true; + return null; + } + try { + if (Files.size(state) > 4096L + (long) maximumEntries * 8192L) { + throw new IOException("Checkpoint exceeds the bounded metadata size limit."); + } + JsonNode contents = mapper.readTree(Files.readAllBytes(state)); + Checkpoint checkpoint = mapper.treeToValue(contents, Checkpoint.class); + if (!contents.path("version").isIntegralNumber() + || (checkpoint.version != 1 && checkpoint.version != 2) + || !watchId.equals(checkpoint.watchId) + || checkpoint.root == null + || checkpoint.scope == null + || (checkpoint.version == 2 + && (!"OBSERVED".equals(checkpoint.semantics) + || !contents.path("generation").isIntegralNumber() + || !contents.path("savedAt").isIntegralNumber() + || checkpoint.generation < 1 + || checkpoint.savedAt < 0)) + || checkpoint.files == null + || checkpoint.files.size() > maximumEntries) { + throw new IOException( + "Unsupported checkpoint version, invalid metadata or entry limit exceeded."); + } + if (root != null && !root.equals(checkpoint.root)) { + throw new IOException( + "Directory differs from the saved checkpoint. Restore the previous Directory or use a new Watch ID for the new location."); + } + if (scope != null && !scope.equals(checkpoint.scope)) { + throw new IOException( + "Include/exclude filters or Include subdirectories differ from the saved checkpoint. Restore the previous options or use a new Watch ID for the new scope."); + } + for (Map.Entry entry : checkpoint.files.entrySet()) { + FileState file = entry.getValue(); + JsonNode fileNode = contents.path("files").path(entry.getKey()); + if (!fileNode.path("size").isIntegralNumber() + || !fileNode.path("lastModified").isIntegralNumber() + || file == null + || !entry.getKey().equals(file.getUri()) + || file.getFilename() == null + || file.getShortFilename() == null + || file.getPath() == null + || file.getScheme() == null + || file.getSize() < 0 + || file.getLastModified() < 0) { + throw new IOException("Invalid file entry."); + } + } + loaded = true; + recoveryRequired = false; + root = checkpoint.root; + scope = checkpoint.scope; + loadedVersion = checkpoint.version; + generation = checkpoint.generation; + return new HashMap<>(checkpoint.files); + } catch (IOException | RuntimeException e) { + recoveryRequired = true; + throw new IOException( + "Cannot load Watch Files state at " + + state + + ": " + + e.getMessage() + + " State has not been overwritten. Use the Maintenance tab to inspect, restore or reset it, or use a new Watch ID explicitly.", + e); + } + } + + @Override + public void save(Map files) throws IOException { + if (recoveryRequired) { + throw new IOException("Invalid checkpoint requires explicit recovery before another save."); + } + if (!loaded) { + load(); + } + if (files.size() > maximumEntries) { + throw new IOException("State entry limit exceeded."); + } + Checkpoint checkpoint = new Checkpoint(); + if (Files.exists(journal)) { + throw new IOException( + "Interrupted checkpoint requires explicit recovery before another save."); + } + if (root == null || scope == null) { + throw new IOException("Cannot save a checkpoint without its root and filter identity."); + } + if (loadedVersion == 1) { + WatchFilesStateManager.archive(state.getParent(), watchId); + } + checkpoint.version = 2; + checkpoint.semantics = "OBSERVED"; + checkpoint.generation = Math.addExact(generation, 1); + checkpoint.savedAt = System.currentTimeMillis(); + checkpoint.watchId = watchId; + checkpoint.root = root; + checkpoint.scope = scope; + checkpoint.files = files; + writeCheckpoint(temporary, mapper.writeValueAsBytes(checkpoint)); + replaceCheckpoint(); + forceDirectory(); + loadedVersion = 2; + generation = checkpoint.generation; + } + + /** Kept overridable so fault tests can simulate ENOSPC without filling the developer's disk. */ + protected void writeCheckpoint(Path target, byte[] contents) throws IOException { + writeForced(target, contents); + } + + protected void atomicReplace() throws IOException { + Files.move( + temporary, state, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } + + protected void nonAtomicReplace() throws IOException { + Files.move(temporary, state, StandardCopyOption.REPLACE_EXISTING); + } + + private void replaceCheckpoint() throws IOException { + try { + atomicReplaceWithRetry(); + } catch (AtomicMoveNotSupportedException e) { + // Preserve a forced known-good backup and durable intent before any non-atomic replacement. + // Recovery is explicit: a leftover intent must never be treated as a fresh first run. + if (Files.exists(state)) { + Files.copy(state, backup, StandardCopyOption.REPLACE_EXISTING); + try (FileChannel channel = FileChannel.open(backup, StandardOpenOption.WRITE)) { + channel.force(true); + } + } + writeForced(journal, new byte[] {1}); + forceDirectory(); + nonAtomicReplace(); + try (FileChannel channel = FileChannel.open(state, StandardOpenOption.WRITE)) { + channel.force(true); + } + forceDirectory(); + Files.delete(journal); + } + } + + private void atomicReplaceWithRetry() throws IOException { + // Windows readers and antivirus software can briefly deny replacement of an open destination. + // Retrying the atomic operation preserves the old checkpoint; never downgrade access failures + // to copy/delete. A persistent permission problem still fails within a bounded 375 ms delay. + for (int attempt = 0; ; attempt++) { + try { + atomicReplace(); + return; + } catch (AccessDeniedException e) { + if (attempt == 4) { + throw e; + } + try { + Thread.sleep(25L << attempt); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + InterruptedIOException failure = + new InterruptedIOException("Checkpoint replacement interrupted."); + failure.initCause(interrupted); + failure.addSuppressed(e); + throw failure; + } + } + } + } + + private static void writeForced(Path target, byte[] contents) throws IOException { + try (FileChannel channel = + FileChannel.open( + target, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.WRITE)) { + ByteBuffer buffer = ByteBuffer.wrap(contents); + while (buffer.hasRemaining()) { + channel.write(buffer); + } + channel.force(true); + } + } + + private void forceDirectory() { + // Windows and some providers cannot open/force a directory. File forcing is still mandatory. + try (FileChannel channel = FileChannel.open(state.getParent(), StandardOpenOption.READ)) { + channel.force(true); + } catch (IOException | UnsupportedOperationException ignored) { + // Best effort directory durability; see the documented power-loss limitation. + } + } + + @Override + public void close() throws IOException { + try { + lock.release(); + } finally { + lockChannel.close(); + } + } + + @Getter + @Setter + public static class Checkpoint { + private int version; + private String watchId; + private String root; + private String scope; + private Map files; + private String semantics; + private long generation; + private long savedAt; + } +} diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/LocalFileWatcher.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/LocalFileWatcher.java new file mode 100644 index 00000000000..4776ef59a09 --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/LocalFileWatcher.java @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import static java.nio.file.StandardWatchEventKinds.ENTRY_CREATE; +import static java.nio.file.StandardWatchEventKinds.ENTRY_DELETE; +import static java.nio.file.StandardWatchEventKinds.ENTRY_MODIFY; +import static java.nio.file.StandardWatchEventKinds.OVERFLOW; + +import java.io.IOException; +import java.nio.file.ClosedWatchServiceException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.WatchEvent; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +/** No worker threads. One WatchService for the whole tree; closing it wakes the Hop runner. */ +public class LocalFileWatcher implements FileWatcher { + private final Path root; + private final boolean recursive; + private final int capacity; + private final int maximumDirectories; + private final WatchService service; + private final Map keys = new HashMap<>(); + private final Set registered = new HashSet<>(); + private volatile boolean closed; + + public LocalFileWatcher(Path root, boolean recursive, int capacity, int maximumDirectories) + throws IOException { + this.root = root; + this.recursive = recursive; + this.capacity = capacity; + this.maximumDirectories = maximumDirectories; + service = root.getFileSystem().newWatchService(); + try { + reconcileDirectories(); + } catch (IOException | RuntimeException e) { + service.close(); + throw e; + } + } + + @Override + public void reconcileDirectories() throws IOException { + if (closed) { + return; + } + if (!recursive) { + register(root); + return; + } + // No FOLLOW_LINKS: recursive symlinks and junctions cannot create loops. + Files.walkFileTree( + root, + new SimpleFileVisitor<>() { + @Override + public FileVisitResult preVisitDirectory(Path directory, BasicFileAttributes attributes) + throws IOException { + register(directory); + return FileVisitResult.CONTINUE; + } + }); + } + + private void register(Path directory) throws IOException { + if (registered.contains(directory)) { + return; + } + if (registered.size() >= maximumDirectories) { + throw new WatchLimitException("Watch directory limit exceeded. Increase maximum entries."); + } + WatchKey key = directory.register(service, ENTRY_CREATE, ENTRY_MODIFY, ENTRY_DELETE); + keys.put(key, directory); + registered.add(directory); + } + + @Override + public WatchBatch poll(long timeoutMillis) throws IOException, InterruptedException { + Set paths = new LinkedHashSet<>(); + boolean reconcile = false; + long overflows = 0; + try { + WatchKey key = service.poll(timeoutMillis, TimeUnit.MILLISECONDS); + // Limit keys per iteration so storms cannot starve stability/checkpoints/stop. + int remaining = capacity; + while (key != null && remaining-- > 0 && !closed) { + Path directory = keys.get(key); + for (WatchEvent event : key.pollEvents()) { + if (event.kind() == OVERFLOW) { + overflows++; + reconcile = true; + continue; + } + if (directory == null || !(event.context() instanceof Path relative)) { + reconcile = true; + continue; + } + Path path = directory.resolve(relative).normalize(); + if (Files.isDirectory(path, java.nio.file.LinkOption.NOFOLLOW_LINKS)) { + // Creation/rename of a populated directory needs a scan, not just registration. + if (recursive) { + reconcile = true; + } + } else if (paths.size() < capacity) { + paths.add(path); + } else { + reconcile = true; + } + if (event.kind() == ENTRY_DELETE && registered.contains(path)) { + reconcile = true; + } + } + if (!resetKey(key)) { + reconcile = true; + } + key = service.poll(); + } + // A key already removed from the service must be processed on the next scan. + if (key != null) { + key.pollEvents(); + resetKey(key); + reconcile = true; + } + } catch (ClosedWatchServiceException e) { + if (!closed) { + throw e; + } + } + return new WatchBatch(paths, reconcile, overflows); + } + + private boolean resetKey(WatchKey key) { + if (key == null) { + return false; + } + if (key.reset()) { + return true; + } + Path removed = keys.remove(key); + if (removed != null) { + registered.remove(removed); + } + return false; + } + + @Override + public void close() throws IOException { + closed = true; + service.close(); + } +} diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/VfsFileScanner.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/VfsFileScanner.java new file mode 100644 index 00000000000..b71970871c2 --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/VfsFileScanner.java @@ -0,0 +1,331 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import java.io.IOException; +import java.net.URI; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryIteratorException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.function.BooleanSupplier; +import java.util.regex.Pattern; +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileType; +import org.apache.commons.vfs2.provider.UriParser; +import org.apache.commons.vfs2.provider.local.LocalFile; +import org.apache.commons.vfs2.provider.local.LocalFileName; +import org.apache.commons.vfs2.util.URIUtils; +import org.apache.hop.core.exception.HopFileException; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.vfs.HopVfs; + +/** A failed or cancelled traversal never returns a partial snapshot. */ +public class VfsFileScanner { + private final FileObject root; + private final boolean recursive; + private final Pattern include; + private final Pattern exclude; + private final int maximumEntries; + private final BooleanSupplier stopped; + + public VfsFileScanner( + FileObject root, + boolean recursive, + String include, + String exclude, + int maximumEntries, + BooleanSupplier stopped) { + this.root = root; + this.recursive = recursive; + this.include = include == null || include.isEmpty() ? null : Pattern.compile(include); + this.exclude = exclude == null || exclude.isEmpty() ? null : Pattern.compile(exclude); + this.maximumEntries = maximumEntries; + this.stopped = stopped; + } + + public Map snapshot() throws IOException { + root.refresh(); + if (!root.exists() || !root.isFolder() || !root.isReadable()) { + throw new IOException( + "Watch root is missing, unreadable or not a directory: " + HopVfs.getFriendlyURI(root)); + } + Map files = new HashMap<>(); + Deque directories = new ArrayDeque<>(); + directories.add("."); + Set visited = new HashSet<>(); + while (!directories.isEmpty()) { + try (FileObject directory = resolveRelative(directories.removeFirst())) { + scan(directory, files, visited, directories); + } + } + return files; + } + + private void scan( + FileObject directory, + Map files, + Set visited, + Deque directories) + throws IOException { + checkStopped(); + if (!visited.add(directory.getName().getURI())) { + return; + } + if (visited.size() > maximumEntries) { + throw new WatchLimitException("Directory entry limit exceeded. Increase maximum entries."); + } + directory.refresh(); + if (!directory.exists() + || !directory.isFolder() + || !directory.isReadable() + || isLink(directory)) { + throw new IOException("Directory changed or became unreadable during reconciliation."); + } + FileObject[] children = children(directory); + if (children == null) { + throw new IOException("VFS provider returned no directory listing."); + } + // Close every handle, including children after the one that failed. + Throwable traversalFailure = null; + try { + for (FileObject child : children) { + checkStopped(); + child.refresh(); + if (isLink(child)) { + continue; + } + FileType type = child.getType(); + if (type == FileType.FOLDER && recursive) { + if (!child.isReadable()) { + throw new IOException("Unreadable child directory."); + } + if (visited.size() + directories.size() >= maximumEntries) { + throw new WatchLimitException( + "Directory entry limit exceeded. Increase maximum entries."); + } + directories.add(root.getName().getRelativeName(child.getName())); + } else if (type == FileType.FILE + && matches(UriParser.decode(child.getName().getBaseName()))) { + FileState file = readState(child); + if (files.size() >= maximumEntries) { + throw new WatchLimitException("File entry limit exceeded. Increase maximum entries."); + } + files.put(file.getUri(), file); + } else if (type == FileType.IMAGINARY) { + // Concurrent removal: restart on the next iteration rather than infer mass deletion. + throw new IOException("Directory changed during listing; retry reconciliation."); + } + } + } catch (IOException | RuntimeException e) { + traversalFailure = e; + throw e; + } finally { + IOException failure = null; + for (FileObject child : children) { + try { + child.close(); + } catch (IOException e) { + if (traversalFailure != null) { + traversalFailure.addSuppressed(e); + } else if (failure == null) { + failure = e; + } else { + failure.addSuppressed(e); + } + } + } + if (failure != null) { + throw failure; + } + } + } + + public FileState readRelative(String relative) throws IOException { + checkStopped(); + try (FileObject file = resolveRelative(relative)) { + file.refresh(); + if (!file.exists() + || file.getType() != FileType.FILE + || isLink(file) + || !matches(UriParser.decode(file.getName().getBaseName()))) { + return null; + } + return readState(file); + } + } + + public String relativeUri(String uri) throws IOException { + if (root instanceof LocalFile) { + String prefix = root.getName().getURI(); + if (!prefix.endsWith("/")) prefix += "/"; + if (!uri.startsWith(prefix)) throw new IOException("File is outside the watched root"); + return uri.substring(prefix.length()); + } + try (FileObject resolved = root.resolveFile(uri)) { + return root.getName().getRelativeName(resolved.getName()); + } + } + + public String uri(String relative) throws IOException { + try (FileObject file = resolveRelative(relative)) { + return file.getName().getURI(); + } + } + + public boolean matches(String name) { + return (include == null || include.matcher(name).matches()) + && (exclude == null || !exclude.matcher(name).matches()); + } + + private FileObject resolveRelative(String relative) throws IOException { + if (!(root instanceof LocalFile)) return root.resolveFile(relative); + LocalFileName name = (LocalFileName) root.getName(); + String prefix = name.getPath(); + if (!prefix.endsWith("/")) prefix += "/"; + return root.getFileSystem() + .resolveFile( + relative.equals(".") ? name : name.createName(prefix + relative, FileType.FILE)); + } + + private FileObject[] children(FileObject directory) throws IOException { + if (!(directory instanceof LocalFile)) return directory.getChildren(); + // VFS 2.10 getChildren() treats a literal Unix backslash as a separator and rejects the + // entire listing. Enumerate local names with NIO, then keep metadata/content access in VFS. + ArrayList children = new ArrayList<>(); + LocalFileName name = (LocalFileName) directory.getName(); + String prefix = name.getPath(); + if (!prefix.endsWith("/")) prefix += "/"; + try (var entries = Files.newDirectoryStream(localPath(directory))) { + for (Path entry : entries) { + checkStopped(); + children.add( + directory + .getFileSystem() + .resolveFile( + name.createName( + prefix + UriParser.encode(entry.getFileName().toString()), FileType.FILE))); + } + } catch (IOException | RuntimeException e) { + for (FileObject child : children) { + try { + child.close(); + } catch (IOException closeFailure) { + e.addSuppressed(closeFailure); + } + } + if (e instanceof DirectoryIteratorException iteration) { + IOException cause = iteration.getCause(); + for (Throwable suppressed : e.getSuppressed()) cause.addSuppressed(suppressed); + throw cause; + } + throw e; + } + return children.toArray(FileObject[]::new); + } + + private FileState readState(FileObject file) throws IOException { + long size = file.getContent().getSize(); + long modified = file.getContent().getLastModifiedTime(); + if (size < 0 || modified < 0) { + throw new IOException("VFS provider must supply a non-negative size and modification time."); + } + String filename = HopVfs.getFilename(file); + if (file instanceof LocalFile && filename.contains("%")) { + // A literal percent in a raw local filename is otherwise interpreted as a VFS escape. + filename = file.getName().getURI(); + } + return new FileState( + file.getName().getURI(), + filename, + UriParser.decode(file.getName().getBaseName()), + file.getName().getParent().getURI(), + file.getName().getScheme(), + size, + modified); + } + + private boolean isLink(FileObject file) throws IOException { + return file.isSymbolicLink() + || (file instanceof LocalFile && Files.isSymbolicLink(localPath(file))); + } + + public static Path localPath(FileObject file) throws IOException { + LocalFileName name = (LocalFileName) file.getName(); + // Match LocalFile.doAttach(): the provider root carries the Windows drive/UNC share, + // while getPathDecoded() preserves literal filename characters without URI parsing. + return Path.of(name.getRootFile() + name.getPathDecoded()).toAbsolutePath().normalize(); + } + + public static FileObject resolveFile(String filename, IVariables variables) + throws HopFileException { + filename = variables.resolve(filename); + // Standard file URIs use UTF-8 percent encoding. Some VFS local parsers decode each escaped + // byte as a character instead, turning an escaped accented filename into a different path. + // Decode explicit file names once without interpreting filename characters as URI syntax. + // URLDecoder has form semantics, so protect literal '+' before UTF-8 percent decoding. + // Remote providers retain their own parsing and Hop's VFS execution context. + if (filename.regionMatches(true, 0, "file:", 0, 5)) { + try { + filename = URLDecoder.decode(filename.replace("+", "%2B"), StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + throw new HopFileException("Invalid percent encoding in local file name", e); + } + // Encode the entire decoded path with VFS's URI codec, not a selected-character list. + // NIO then interprets the platform drive/UNC root at this explicit URI boundary only. + return resolveLocalPath(Path.of(URI.create(URIUtils.encodePath(filename))), variables); + } else { + String scheme = UriParser.extractScheme(filename); + if (scheme == null || (scheme.length() == 1 && Character.isLetter(scheme.charAt(0)))) { + return resolveLocalPath(Path.of(filename), variables); + } + } + return HopVfs.getFileObject(filename, variables); + } + + private static FileObject resolveLocalPath(Path path, IVariables variables) + throws HopFileException { + Path absolute = path.toAbsolutePath().normalize(); + Path prefix = absolute.getRoot(); + try (FileObject rootFile = + HopVfs.getFileObject(UriParser.encode(prefix.toString()), variables)) { + LocalFileName name = (LocalFileName) rootFile.getName(); + String relative = + prefix.relativize(absolute).toString().replace(path.getFileSystem().getSeparator(), "/"); + return rootFile + .getFileSystem() + .resolveFile(name.createName("/" + UriParser.encode(relative), FileType.FILE)); + } catch (IOException e) { + throw new HopFileException("Unable to resolve local filesystem path", e); + } + } + + private void checkStopped() throws IOException { + if (stopped.getAsBoolean()) { + throw new IOException("Watch Files was stopped."); + } + } +} diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/VfsPollingWatcher.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/VfsPollingWatcher.java new file mode 100644 index 00000000000..2a5fe9c5f3b --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/VfsPollingWatcher.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import java.util.Set; + +/** + * Interruptible idle wait; listing stays on the transform thread with its VFS execution context. + */ +public class VfsPollingWatcher implements FileWatcher { + private boolean closed; + + @Override + public synchronized WatchBatch poll(long timeoutMillis) throws InterruptedException { + if (!closed && timeoutMillis > 0) { + wait(timeoutMillis); + } + return new WatchBatch(Set.of(), false, 0); + } + + @Override + public synchronized void close() { + closed = true; + notifyAll(); + } +} diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFiles.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFiles.java new file mode 100644 index 00000000000..c02ea41dbfd --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFiles.java @@ -0,0 +1,347 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Date; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.provider.local.LocalFile; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.row.RowMeta; +import org.apache.hop.core.vfs.HopVfs; +import org.apache.hop.pipeline.Pipeline; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransform; +import org.apache.hop.pipeline.transform.TransformMeta; + +public class WatchFiles extends BaseTransform { + private final AtomicBoolean stopping = new AtomicBoolean(); + + public WatchFiles( + TransformMeta transformMeta, + WatchFilesMeta meta, + WatchFilesData data, + int copyNr, + PipelineMeta pipelineMeta, + Pipeline pipeline) { + super(transformMeta, meta, data, copyNr, pipelineMeta, pipeline); + } + + @Override + public boolean init() { + if (!super.init()) { + return false; + } + FileWatcher watcher = null; + FileStateStore store = null; + try { + meta.validate(this); + data.maximumRunMillis = meta.maximumRunMillis(this); + if (getPipeline().getPipelineType() != PipelineMeta.PipelineType.Normal) { + throw new IOException("Watch Files requires the normal Local Hop Engine."); + } + if (getCopy() != 0 || getTransformMeta().getCopies(this) != 1) { + throw new IOException("Watch Files requires exactly one transform copy."); + } + if (!getPipelineMeta().findPreviousTransforms(getTransformMeta()).isEmpty()) { + throw new IOException("Watch Files is a source and does not accept input rows."); + } + data.outputRowMeta = new RowMeta(); + meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); + data.watchId = resolve(meta.getWatchId()); + data.root = VfsFileScanner.resolveFile(resolve(meta.getDirectory()), this); + if (!data.root.exists() || !data.root.isFolder() || !data.root.isReadable()) { + throw new IOException("Watch root must be an existing readable directory."); + } + if (data.root.isSymbolicLink()) { + throw new IOException("Watch root must not be a symbolic link."); + } + Path localRoot = + data.root instanceof LocalFile && "file".equals(data.root.getName().getScheme()) + ? VfsFileScanner.localPath(data.root) + : null; + if (localRoot != null) { + if (Files.isSymbolicLink(localRoot)) { + throw new IOException("Watch root must not be a symbolic link."); + } + localRoot = localRoot.toRealPath(); + } + Path statePath; + try (FileObject stateFolder = + VfsFileScanner.resolveFile(resolve(meta.getStateDirectory()), this)) { + if (!(stateFolder instanceof LocalFile)) { + throw new IOException( + "State directory must be local; remote checkpoint stores are not supported."); + } + stateFolder.createFolder(); + statePath = VfsFileScanner.localPath(stateFolder).toRealPath(); + } + if (localRoot != null && statePath.startsWith(localRoot)) { + throw new IOException("State directory must be outside the watched tree."); + } + int maximum = WatchFilesMeta.integer(this, meta.getMaximumEntries(), "Maximum entries"); + int capacity = WatchFilesMeta.integer(this, meta.getEventCapacity(), "Native hint capacity"); + DetectionStrategy strategy = DetectionStrategy.valueOf(meta.getStrategy()); + if (strategy == DetectionStrategy.NATIVE && localRoot == null) { + throw new IOException( + "NATIVE requires a local file filesystem. Use AUTO or POLLING for VFS."); + } + if (strategy != DetectionStrategy.POLLING && localRoot != null) { + try { + watcher = + new LocalFileWatcher(localRoot, meta.isIncludeSubdirectories(), capacity, maximum); + } catch (IOException | UnsupportedOperationException e) { + if (e instanceof WatchLimitException) { + throw e; + } + if (strategy == DetectionStrategy.NATIVE) { + throw e; + } + logBasic("Native watcher unavailable; using VFS polling: " + e.getMessage()); + } + } + boolean nativeWatcher = watcher != null; + if (!nativeWatcher) { + watcher = new VfsPollingWatcher(); + } + String include = meta.filenameRegex(this, meta.getIncludeWildcard()); + String exclude = meta.filenameRegex(this, meta.getExcludeWildcard()); + String scope = + new ObjectMapper() + .writeValueAsString( + new String[] { + Boolean.toString(meta.isIncludeSubdirectories()), include, exclude + }); + store = + new JsonFileStateStore( + statePath, data.watchId, HopVfs.getFriendlyURI(data.root), scope, maximum); + VfsFileScanner scanner = + new VfsFileScanner( + data.root, meta.isIncludeSubdirectories(), include, exclude, maximum, stopping::get); + java.util.function.Consumer watchLog = + message -> logBasic("[watch_id=" + data.watchId + "] " + message); + WatchFilesDiagnostics diagnostics = + new WatchFilesDiagnostics( + statePath.resolve(data.watchId).toString(), + data.watchId, + WatchFilesClock.SYSTEM, + WatchFilesMeta.number( + this, meta.getSlowOperationThreshold(), 1, "Slow operation threshold"), + WatchFilesMeta.number(this, meta.getDiagnosticsInterval(), 1, "Diagnostics interval"), + watchLog); + data.engine = + new WatchFilesEngine( + scanner, + watcher, + store, + new FileStabilityTracker( + meta.isWaitUntilStable(), + WatchFilesMeta.number(this, meta.getMinimumAge(), 0, "Minimum age"), + WatchFilesMeta.integer(this, meta.getStabilityChecks(), "Stability checks"), + WatchFilesMeta.number(this, meta.getStabilityInterval(), 1, "Stability interval"), + maximum), + WatchFilesMeta.number( + this, + nativeWatcher ? meta.getReconciliationInterval() : meta.getPollingInterval(), + 1, + "Scan interval"), + WatchFilesMeta.number(this, meta.getCheckpointInterval(), 1, "Checkpoint interval"), + WatchFilesMeta.number(this, meta.getPollingInterval(), 1, "Retry interval"), + nativeWatcher ? localRoot : null, + maximum, + watchLog, + WatchFilesClock.SYSTEM, + diagnostics); + data.engine.initialize(InitialScan.valueOf(meta.getInitialScan())); + diagnostics.register(); + if (stopping.get()) { + data.engine.stop(); + } + logBasic( + "Watch Files initialized for [" + + HopVfs.getFriendlyURI(data.root) + + "], using " + + (nativeWatcher ? "native WatchService" : "VFS polling") + + "."); + return true; + } catch (Exception e) { + logError("Unable to initialize Watch Files: " + e.getMessage(), e); + closeFailedInitialization(watcher, store); + return false; + } + } + + private void closeFailedInitialization(FileWatcher watcher, FileStateStore store) { + try { + if (data.engine != null) { + data.engine.close(); + } else { + try { + if (watcher != null) watcher.close(); + } finally { + if (store != null) store.close(); + } + } + } catch (IOException e) { + logError("Unable to close watcher and release state lock", e); + } + data.engine = null; + try { + if (data.root != null) { + data.root.close(); + } + } catch (IOException e) { + logError("Unable to close VFS root", e); + } + data.root = null; + } + + @Override + public boolean processRow() throws HopException { + if (isStopped() || stopping.get()) { + setOutputDone(); + return false; + } + try { + if (!data.runStarted) { + data.runStartedAt = WatchFilesClock.SYSTEM.elapsedMillis(); + data.runStarted = true; + } + if (finishIfRuntimeExpired()) return false; + data.engine.tick(); + if (finishIfRuntimeExpired()) return false; + FileChangeEvent event = data.engine.next(); + if (finishIfRuntimeExpired()) return false; + if (event != null) { + if (meta.enabled(event.getType())) { + putRow(data.outputRowMeta, toRow(event, data.watchId)); + // putRow can return after a stop without handing the row downstream. + if (isStopped() || stopping.get()) { + setOutputDone(); + return false; + } + incrementLinesInput(); + } + data.engine.acknowledge(event, meta.enabled(event.getType())); + data.engine.checkpoint(); + } else { + long waitMillis = 100; + if (data.maximumRunMillis > 0) { + waitMillis = + Math.min( + waitMillis, + Math.max( + 1, + data.maximumRunMillis + - (WatchFilesClock.SYSTEM.elapsedMillis() - data.runStartedAt))); + } + data.engine.await(waitMillis); + } + return !isStopped() && !stopping.get(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + if (isStopped() || stopping.get()) { + setOutputDone(); + return false; + } + throw new HopException("Watch Files interrupted.", e); + } catch (IOException | RuntimeException e) { + if (isStopped() || stopping.get()) { + setOutputDone(); + return false; + } + throw new HopException("Watch Files failed; acknowledged checkpoint is retained.", e); + } + } + + private boolean finishIfRuntimeExpired() throws IOException { + if (data.maximumRunMillis == 0 + || WatchFilesClock.SYSTEM.elapsedMillis() - data.runStartedAt < data.maximumRunMillis) { + return false; + } + // Close and persist acknowledged observations before signalling normal EOF downstream. + // Do not mark the pipeline stopped: queued rows must be allowed to finish. + data.engine.close(); + data.engine = null; + logBasic("[watch_id=" + data.watchId + "] Maximum run time reached; state saved."); + setOutputDone(); + return true; + } + + static Object[] toRow(FileChangeEvent event, String watchId) { + FileState file = event.file(); + FileState previous = event.getPrevious(); + return new Object[] { + file.getFilename(), + file.getShortFilename(), + file.getPath(), + file.getUri(), + event.getType().name(), + file.getSize(), + new Date(file.getLastModified()), + new Date(event.getDetectedAt()), + false, + file.getScheme(), + previous == null ? null : previous.getSize(), + previous == null ? null : new Date(previous.getLastModified()), + watchId + }; + } + + @Override + public void stopRunning() throws HopException { + stopping.set(true); + try { + WatchFilesEngine engine = data.engine; + if (engine != null) { + engine.stop(); + } + } catch (IOException e) { + throw new HopException("Unable to stop Watch Files.", e); + } finally { + super.stopRunning(); + } + } + + @Override + public void dispose() { + try { + if (data.engine != null) { + data.engine.close(); + data.engine = null; + } + } catch (IOException e) { + logError("Unable to checkpoint Watch Files during disposal.", e); + setErrors(getErrors() + 1); + } finally { + try { + if (data.root != null) { + data.root.close(); + data.root = null; + } + } catch (IOException e) { + logError("Unable to close VFS root.", e); + } + super.dispose(); + } + } +} diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesClock.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesClock.java new file mode 100644 index 00000000000..bd6abfeb59a --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesClock.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import java.util.concurrent.TimeUnit; + +/** Separate filesystem timestamps from elapsed time, which must survive wall-clock adjustments. */ +public interface WatchFilesClock { + WatchFilesClock SYSTEM = + new WatchFilesClock() { + private final long origin = System.nanoTime(); + + @Override + public long elapsedMillis() { + return TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - origin); + } + + @Override + public long wallMillis() { + return System.currentTimeMillis(); + } + }; + + long elapsedMillis(); + + long wallMillis(); +} diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesData.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesData.java new file mode 100644 index 00000000000..2dd7bc576f9 --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesData.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import org.apache.commons.vfs2.FileObject; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.pipeline.transform.BaseTransformData; + +public class WatchFilesData extends BaseTransformData { + public IRowMeta outputRowMeta; + public volatile WatchFilesEngine engine; + public FileObject root; + public String watchId; + public long maximumRunMillis; + public long runStartedAt; + public boolean runStarted; +} diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesDiagnostics.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesDiagnostics.java new file mode 100644 index 00000000000..eafa1a9d02c --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesDiagnostics.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Consumer; +import lombok.Getter; + +/** Counters have one writer; readers see an immutable, volatile snapshot. No monitoring thread. */ +public class WatchFilesDiagnostics { + private static final ConcurrentHashMap ACTIVE = + new ConcurrentHashMap<>(); + private final String key; + private final String watchId; + private final WatchFilesClock clock; + private final Consumer log; + private final long slowMillis; + private final long reportInterval; + private long nextReport; + private long scans; + private long checkpoints; + private long retries; + private long overflow; + private long emitted; + private long suppressed; + private long lastScanAt; + private long scanMillis; + private long checkpointMillis; + private String lastFailure = ""; + @Getter private volatile Snapshot snapshot; + + public WatchFilesDiagnostics( + String key, + String watchId, + WatchFilesClock clock, + long slowMillis, + long reportInterval, + Consumer log) { + this.key = key; + this.watchId = watchId; + this.clock = clock; + this.slowMillis = slowMillis; + this.reportInterval = reportInterval; + this.log = log; + nextReport = clock.elapsedMillis() + reportInterval; + publish(0, 0); + } + + public void register() { + if (ACTIVE.putIfAbsent(key, this) != null) { + throw new IllegalStateException("Diagnostics already registered for Watch ID " + watchId); + } + } + + public static Snapshot active(String key) { + WatchFilesDiagnostics diagnostics = ACTIVE.get(key); + return diagnostics == null ? null : diagnostics.getSnapshot(); + } + + public void unregister() { + ACTIVE.remove(key, this); + } + + public void scan(long start) { + scans++; + lastScanAt = clock.wallMillis(); + scanMillis = duration("scan", start); + } + + public void checkpoint(long start) { + checkpoints++; + checkpointMillis = duration("checkpoint", start); + } + + public void probe(long start) { + duration("stat", start); + } + + public void failure(String operation) { + retries++; + lastFailure = operation; + } + + public void overflow(long count) { + overflow += count; + } + + public void acknowledged(boolean enabled) { + if (enabled) emitted++; + else suppressed++; + } + + private long duration(String operation, long start) { + long millis = Math.max(0, clock.elapsedMillis() - start); + if (millis >= slowMillis) log.accept("Slow " + operation + ": duration_ms=" + millis); + return millis; + } + + public void publish(int observed, int pending) { + snapshot = + new Snapshot( + watchId, + clock.wallMillis(), + observed, + pending, + scans, + checkpoints, + retries, + overflow, + emitted, + suppressed, + lastScanAt, + scanMillis, + checkpointMillis, + lastFailure); + } + + public void report(int observed, int pending, boolean force) { + publish(observed, pending); + long now = clock.elapsedMillis(); + if (force || now >= nextReport) { + log.accept("Diagnostics " + snapshot); + nextReport = now + reportInterval; + } + } + + public record Snapshot( + String watchId, + long sampledAt, + int observed, + int pending, + long scans, + long checkpoints, + long retries, + long overflow, + long emitted, + long suppressed, + long lastSuccessfulScanAt, + long scanDurationMillis, + long checkpointDurationMillis, + String lastFailure) {} +} diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesDialog.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesDialog.java new file mode 100644 index 00000000000..ad8aae5d354 --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesDialog.java @@ -0,0 +1,372 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.ui.core.dialog.BaseDialog; +import org.apache.hop.ui.core.dialog.ProgressMonitorDialog; +import org.apache.hop.ui.core.gui.GuiCompositeWidgets; +import org.apache.hop.ui.core.gui.IGuiPluginCompositeWidgetsListener; +import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; +import org.eclipse.swt.SWT; +import org.eclipse.swt.layout.FormAttachment; +import org.eclipse.swt.layout.FormData; +import org.eclipse.swt.layout.GridData; +import org.eclipse.swt.layout.GridLayout; +import org.eclipse.swt.widgets.Button; +import org.eclipse.swt.widgets.Composite; +import org.eclipse.swt.widgets.Control; +import org.eclipse.swt.widgets.FileDialog; +import org.eclipse.swt.widgets.Label; +import org.eclipse.swt.widgets.MessageBox; +import org.eclipse.swt.widgets.Shell; +import org.eclipse.swt.widgets.Text; + +public class WatchFilesDialog extends BaseTransformDialog { + private final WatchFilesMeta input; + private GuiCompositeWidgets widgets; + private Label patternHelp; + private Label stateIdentity; + + public WatchFilesDialog( + Shell parent, IVariables variables, WatchFilesMeta transformMeta, PipelineMeta pipelineMeta) { + super(parent, variables, transformMeta, pipelineMeta); + input = transformMeta; + } + + @Override + public String open() { + createShell(BaseMessages.getString(WatchFilesMeta.class, "WatchFilesDialog.Title")); + buildButtonBar().ok(e -> ok()).cancel(e -> cancel()).build(); + changed = input.hasChanged(); + WatchFilesMeta form = (WatchFilesMeta) input.clone(); + form.setInitialScan(WatchFilesMeta.optionLabel("initialScan", input.getInitialScan())); + form.setStrategy(WatchFilesMeta.optionLabel("strategy", input.getStrategy())); + form.setPatternSyntax(WatchFilesMeta.optionLabel("patternSyntax", input.getPatternSyntax())); + form.setMaximumRunTimeUnit( + WatchFilesMeta.optionLabel("maximumRunTimeUnit", input.getMaximumRunTimeUnit())); + widgets = + GuiCompositeWidgets.addScrolledComposite( + shell, + variables, + wTransformName, + wOk, + WatchFilesMeta.GUI_PLUGIN_ELEMENT_PARENT_ID, + form, + groups -> { + groups.registerExtraGroup(group("General"), "01", null, this::generalHelp); + groups.registerExtraGroup(group("Maintenance"), "03", null, this::stateTools); + }); + widgets.setCompositeWidgetsListener( + new IGuiPluginCompositeWidgetsListener() { + @Override + public void widgetsCreated(GuiCompositeWidgets compositeWidgets) {} + + @Override + public void widgetsPopulated(GuiCompositeWidgets compositeWidgets) {} + + @Override + public void widgetModified( + GuiCompositeWidgets compositeWidgets, Control control, String id) { + input.setChanged(); + updateOptions(); + } + + @Override + public void persistContents(GuiCompositeWidgets compositeWidgets) {} + }); + updateOptions(); + Control viewport = widgets.getWidgetsMap().get("directory").getParent(); + while (!(viewport instanceof org.eclipse.swt.custom.ScrolledComposite) + || viewport.getParent() != shell) { + viewport = viewport.getParent(); + } + FormData viewportLayout = (FormData) viewport.getLayoutData(); + viewportLayout.width = 760; + viewportLayout.height = 420; + focusTransformName(); + BaseDialog.defaultShellHandling(shell, c -> ok(), c -> cancel()); + return transformName; + } + + private void ok() { + if (Utils.isEmpty(wTransformName.getText())) { + return; + } + readOptions(input); + transformName = wTransformName.getText(); + input.setChanged(); + dispose(); + } + + private void cancel() { + transformName = null; + input.setChanged(changed); + dispose(); + } + + private String message(String key) { + return BaseMessages.getString(WatchFilesMeta.class, "WatchFilesDialog." + key); + } + + private String group(String key) { + return BaseMessages.getString(WatchFilesMeta.class, "WatchFilesMeta.Group." + key); + } + + private void readOptions(WatchFilesMeta options) { + String previousInitialScan = input.getInitialScan(); + widgets.getWidgetsContents(options, WatchFilesMeta.GUI_PLUGIN_ELEMENT_PARENT_ID); + // IGNORE_EXISTING and COMPARE_WITH_STATE have the same first-start baseline behavior. + // Preserve the persisted choice when its displayed label has not changed. + options.setInitialScan( + java.util.Objects.equals( + WatchFilesMeta.optionLabel("initialScan", previousInitialScan), + options.getInitialScan()) + ? previousInitialScan + : WatchFilesMeta.optionValue("initialScan", options.getInitialScan())); + options.setMaximumRunTimeUnit( + WatchFilesMeta.optionValue("maximumRunTimeUnit", options.getMaximumRunTimeUnit())); + options.setStrategy(WatchFilesMeta.optionValue("strategy", options.getStrategy())); + options.setPatternSyntax( + WatchFilesMeta.optionValue("patternSyntax", options.getPatternSyntax())); + } + + private void updateOptions() { + WatchFilesMeta options = (WatchFilesMeta) input.clone(); + readOptions(options); + Set hidden = new HashSet<>(); + String duration = variables.resolve(options.getMaximumRunTime()); + if (duration == null || duration.isBlank()) hidden.add("maximumRunTimeUnit"); + if (!options.isEditWatchId()) hidden.add("watchId"); + if (!options.isWaitUntilStable()) { + hidden.addAll(Set.of("stabilityChecks", "stabilityInterval")); + } + if ("POLLING".equals(options.getStrategy())) { + hidden.addAll(Set.of("eventCapacity", "reconciliationInterval")); + } else if ("NATIVE".equals(options.getStrategy())) { + hidden.add("pollingInterval"); + } + widgets.setWidgetsHidden(options, hidden); + setHelp("strategy", "Detection." + options.getStrategy()); + setHelp("pollingInterval", "PollingHelp." + options.getStrategy()); + setHelp("reconciliationInterval", "ReconciliationHelp"); + setHelp("includeWildcard", "IncludeHelp." + options.getPatternSyntax()); + setHelp("excludeWildcard", "ExcludeHelp." + options.getPatternSyntax()); + patternHelp.setText( + message("GeneralHelp") + "\n" + message("Patterns." + options.getPatternSyntax())); + String id = options.getWatchId(); + boolean generated = id != null && id.matches("watch-[a-f0-9]{8}(-[a-f0-9]{4}){3}-[a-f0-9]{12}"); + stateIdentity.setText(message(generated ? "StateIdentity.Automatic" : "StateIdentity.Saved")); + stateIdentity.setToolTipText( + BaseMessages.getString(WatchFilesMeta.class, "WatchFilesDialog.StateIdentityHelp", id)); + } + + private void setHelp(String id, String key) { + String help = message(key); + Control widget = widgets.getWidgetsMap().get(id); + if (widget != null) widget.setToolTipText(help); + Control label = widgets.getLabelsMap().get(id); + if (label != null) label.setToolTipText(help); + } + + private Composite extraPanel(Composite parent) { + Control[] existing = parent.getChildren(); + Composite panel = new Composite(parent, SWT.NONE); + FormData position = new FormData(); + position.left = new FormAttachment(0); + position.right = new FormAttachment(100); + position.top = + existing.length == 0 + ? new FormAttachment(0) + : new FormAttachment(existing[existing.length - 1], 10); + panel.setLayoutData(position); + panel.setLayout(new GridLayout(2, false)); + return panel; + } + + private void generalHelp(Composite parent) { + Composite panel = extraPanel(parent); + stateIdentity = new Label(panel, SWT.NONE); + stateIdentity.setLayoutData(new GridData(SWT.FILL, SWT.CENTER, true, false, 2, 1)); + patternHelp = new Label(panel, SWT.WRAP); + GridData area = new GridData(SWT.FILL, SWT.CENTER, true, false, 2, 1); + area.widthHint = 650; + patternHelp.setLayoutData(area); + patternHelp.setText(message("GeneralHelp")); + } + + private void stateTools(Composite parent) { + parent = extraPanel(parent); + // Other tabs can require a wider scrollable form. Keep these few actions compact. + FormData panelArea = (FormData) parent.getLayoutData(); + panelArea.right = null; + panelArea.width = 650; + Label help = new Label(parent, SWT.WRAP); + GridData helpArea = new GridData(SWT.FILL, SWT.CENTER, true, false, 2, 1); + helpArea.widthHint = 650; + help.setLayoutData(helpArea); + help.setText(message("MaintenanceHelp")); + Text status = + new Text(parent, SWT.MULTI | SWT.READ_ONLY | SWT.BORDER | SWT.WRAP | SWT.V_SCROLL); + GridData area = new GridData(SWT.FILL, SWT.FILL, true, true, 2, 1); + area.heightHint = 100; + area.widthHint = 650; + area.exclude = true; + status.setLayoutData(area); + status.setVisible(false); + actionButton(parent, "Inspect", status, 1); + actionButton(parent, "Backup", status, 1); + actionButton(parent, "Replay", status, 2); + Button advanced = new Button(parent, SWT.CHECK); + advanced.setText(message("AdvancedRecovery")); + advanced.setToolTipText(message("AdvancedRecoveryHelp")); + advanced.setLayoutData(new GridData(SWT.FILL, SWT.CENTER, true, false, 2, 1)); + Composite recovery = new Composite(parent, SWT.NONE); + recovery.setLayout(new GridLayout(2, false)); + GridData recoveryArea = new GridData(SWT.FILL, SWT.CENTER, true, false, 2, 1); + recoveryArea.exclude = true; + recovery.setLayoutData(recoveryArea); + recovery.setVisible(false); + actionButton(recovery, "Reset", status, 1); + actionButton(recovery, "Restore", status, 1); + actionButton(recovery, "Migrate", status, 2); + advanced.addListener( + SWT.Selection, + event -> { + recoveryArea.exclude = !advanced.getSelection(); + recovery.setVisible(advanced.getSelection()); + refreshStatePanel(status); + }); + } + + private void actionButton(Composite parent, String action, Text status, int span) { + Button button = new Button(parent, SWT.PUSH); + button.setText(message(action)); + button.setToolTipText(message(action + "Help")); + button.setLayoutData(new GridData(SWT.FILL, SWT.CENTER, true, false, span, 1)); + button.addListener(SWT.Selection, event -> stateAction(action, status)); + } + + private void refreshStatePanel(Text status) { + Composite panel = status.getParent(); + panel.layout(true, true); + Composite content = panel.getParent(); + content.layout(true, true); + if (content.getParent() instanceof org.eclipse.swt.custom.ScrolledComposite scrolled) { + scrolled.setMinSize(content.computeSize(SWT.DEFAULT, SWT.DEFAULT)); + } + } + + private void showStateResult(Text status, String result) { + status.setText(result); + ((GridData) status.getLayoutData()).exclude = false; + status.setVisible(true); + refreshStatePanel(status); + } + + private void stateAction(String action, Text status) { + try { + WatchFilesMeta options = (WatchFilesMeta) input.clone(); + readOptions(options); + if (action.equals("Replay")) { + if (!options.isCreated()) throw new java.io.IOException(message("CreatedRequired")); + if (!new WatchFilesReplayDialog(shell, variables, options).open()) return; + } + Path directory; + try (org.apache.commons.vfs2.FileObject folder = + VfsFileScanner.resolveFile(variables.resolve(options.getStateDirectory()), variables)) { + if (!(folder instanceof org.apache.commons.vfs2.provider.local.LocalFile)) { + throw new java.io.IOException(message("LocalStateRequired")); + } + directory = VfsFileScanner.localPath(folder); + if (java.nio.file.Files.exists(directory)) directory = directory.toRealPath(); + } + WatchFilesStateManager manager = + new WatchFilesStateManager( + directory, + variables.resolve(options.getWatchId()), + WatchFilesMeta.integer(variables, options.getMaximumEntries(), "Maximum entries")); + Path selected = null; + if (action.equals("Restore")) { + FileDialog chooser = new FileDialog(shell, SWT.OPEN); + chooser.setText(message("Restore")); + chooser.setFilterPath(directory.toString()); + chooser.setFilterExtensions(new String[] {"*.json;*.bak", "*"}); + String filename = chooser.open(); + if (filename == null) return; + selected = Path.of(filename); + } + if (action.equals("Replay") + || action.equals("Reset") + || action.equals("Restore") + || action.equals("Migrate")) { + if (!confirmStateAction(action)) return; + } + final Path restore = selected; + AtomicReference result = new AtomicReference<>(); + new ProgressMonitorDialog(shell) + .run( + false, + monitor -> { + monitor.beginTask(message(action), 1); + try { + result.set( + switch (action) { + case "Inspect" -> manager.inspect(); + case "Backup" -> message("BackupResult") + manager.backup(); + case "Replay" -> + message("ReplayResult") + + manager.replay(variables.resolve(options.getReplayFilter())); + case "Reset" -> message("BackupResult") + manager.reset(); + case "Restore" -> message("BackupResult") + manager.restoreBackup(restore); + case "Migrate" -> { + manager.migrate(); + yield manager.inspect(); + } + default -> throw new IllegalArgumentException(action); + }); + } catch (Exception failure) { + throw new java.lang.reflect.InvocationTargetException(failure); + } finally { + monitor.done(); + } + }); + showStateResult(status, result.get()); + } catch (Exception failure) { + Throwable cause = + failure instanceof java.lang.reflect.InvocationTargetException + ? failure.getCause() + : failure; + showStateResult(status, message("StateError") + cause.getMessage()); + } + } + + protected boolean confirmStateAction(String action) { + MessageBox confirmation = new MessageBox(shell, SWT.ICON_QUESTION | SWT.YES | SWT.NO); + confirmation.setText(message(action)); + confirmation.setMessage(message(action + "Confirm")); + return confirmation.open() == SWT.YES; + } +} diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesEngine.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesEngine.java new file mode 100644 index 00000000000..25aa2e13099 --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesEngine.java @@ -0,0 +1,408 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; +import org.apache.commons.vfs2.provider.UriParser; + +/** + * Single owner: filesystem hints, reconciliation, stability and acknowledgement all run on the Hop + * transform thread. stop() only sets a flag and closes the blocking watcher. + */ +public class WatchFilesEngine implements AutoCloseable { + private final VfsFileScanner scanner; + private final FileWatcher watcher; + private final FileStateStore store; + private final FileStabilityTracker stability; + private final FileReconciler reconciler = new FileReconciler(); + private final Map observed = new HashMap<>(); + private final long scanInterval; + private final long checkpointInterval; + private final long retryInterval; + private final Consumer log; + private final Path localRoot; + private final int maximumEntries; + private final WatchFilesClock clock; + private final WatchFilesDiagnostics diagnostics; + private volatile boolean stopped; + private boolean initialized; + private boolean dirty; + private boolean scanRequired; + private long nextScan; + private long nextCheckpoint; + private long nextProbe; + + public WatchFilesEngine( + VfsFileScanner scanner, + FileWatcher watcher, + FileStateStore store, + FileStabilityTracker stability, + long scanInterval, + long checkpointInterval, + long retryInterval, + Path localRoot, + int maximumEntries, + Consumer log) { + this( + scanner, + watcher, + store, + stability, + scanInterval, + checkpointInterval, + retryInterval, + localRoot, + maximumEntries, + log, + WatchFilesClock.SYSTEM, + null); + } + + public WatchFilesEngine( + VfsFileScanner scanner, + FileWatcher watcher, + FileStateStore store, + FileStabilityTracker stability, + long scanInterval, + long checkpointInterval, + long retryInterval, + Path localRoot, + int maximumEntries, + Consumer log, + WatchFilesClock clock, + WatchFilesDiagnostics diagnostics) { + this.clock = clock; + this.diagnostics = diagnostics; + this.scanner = scanner; + this.watcher = watcher; + this.store = store; + this.stability = stability; + this.scanInterval = scanInterval; + this.checkpointInterval = checkpointInterval; + this.retryInterval = retryInterval; + this.localRoot = localRoot; + this.maximumEntries = maximumEntries; + this.log = log; + } + + public void initialize(InitialScan initial) throws IOException { + initialize(initial, clock.elapsedMillis(), clock.wallMillis()); + } + + /** Explicit times support deterministic engine tests without sleeping. */ + public void initialize(InitialScan initial, long now) throws IOException { + initialize(initial, now, now); + } + + private void initialize(InitialScan initial, long now, long wallTime) throws IOException { + long initializationStarted = clock.elapsedMillis(); + Map saved = store.load(); + Map snapshot = snapshot(); + if (saved != null) { + observed.putAll(saved); + log.accept("Loaded state with " + observed.size() + " entries."); + } else { + if (initial != InitialScan.EMIT_EXISTING) { + // COMPARE_WITH_STATE without a checkpoint establishes an explicit quiet baseline. + observed.putAll(snapshot); + log.accept("Initial baseline contains " + observed.size() + " files."); + } + save(); + } + initialized = true; + reconcile(snapshot, now, wallTime); + long completedAt = now + Math.max(0, clock.elapsedMillis() - initializationStarted); + nextScan = completedAt + scanInterval; + nextCheckpoint = completedAt + checkpointInterval; + report(false); + } + + public void tick() throws IOException, InterruptedException { + tick(clock.elapsedMillis(), clock.wallMillis()); + } + + public void tick(long now) throws IOException, InterruptedException { + tick(now, now); + } + + private void tick(long now, long wallTime) throws IOException, InterruptedException { + if (stopped) { + return; + } + FileWatcher.WatchBatch batch = deferredBatch == null ? watcher.poll(0) : deferredBatch; + deferredBatch = null; + if (batch.getOverflowCount() > 0) { + if (diagnostics != null) diagnostics.overflow(batch.getOverflowCount()); + log.accept("Watch service overflow detected. Scheduling reconciliation scan."); + } + if (batch.isReconciliationRequired()) { + scanRequired = true; + nextScan = now; + } + if (scanRequired || now >= nextScan) { + // Never advance deletion state after a failed/incomplete scan. + long scanStarted = clock.elapsedMillis(); + try { + watcher.reconcileDirectories(); + reconcile(snapshot(), now, wallTime); + scanRequired = false; + nextScan = now + Math.max(0, clock.elapsedMillis() - scanStarted) + scanInterval; + } catch (IOException e) { + if (e instanceof WatchLimitException) { + throw e; + } + if (stopped) { + return; + } + if (diagnostics != null) diagnostics.failure("scan"); + log.accept("Reconciliation failed; state retained. Retrying: " + e.getMessage()); + scanRequired = false; + nextScan = now + Math.max(0, clock.elapsedMillis() - scanStarted) + retryInterval; + } + } else if (localRoot != null) { + for (Path path : batch.getPaths()) { + if (stopped) { + return; + } + if (!path.startsWith(localRoot) || !scanner.matches(path.getFileName().toString())) { + continue; + } + String relative = + UriParser.encode( + localRoot + .relativize(path) + .toString() + .replace(localRoot.getFileSystem().getSeparator(), "/"), + new char[] {'%'}); + try { + FileState current = scanner.readRelative(relative); + // A missing file's canonical VFS URI is found through its persisted relative path. + if (current != null) { + observe(current.getUri(), current, now, wallTime); + } else { + String uri = scanner.uri(relative); + FileState previous = observed.get(uri); + stability.remove(uri); + if (previous != null) { + deletions.put( + uri, new FileChangeEvent(FileEventType.DELETED, null, previous, wallTime)); + } + } + } catch (IOException e) { + scanRequired = true; + } + } + } + checkpoint(now); + report(false); + } + + private void reconcile(Map current, long now, long wallTime) { + reconciler.compare( + observed, + current, + wallTime, + event -> { + if (event.getType() == FileEventType.DELETED) { + stability.remove(event.file().getUri()); + // Deletions use the same pending pipeline as writes, but require no stability wait. + deletions.put(event.file().getUri(), event); + } else { + stability.observe(event, now); + } + }); + // Files can disappear before their first emission, or return to their observed version. + for (String uri : stability.uris()) { + FileState currentFile = current.get(uri); + if (currentFile == null || currentFile.sameVersion(observed.get(uri))) { + stability.remove(uri); + } + } + deletions.keySet().removeIf(current::containsKey); + } + + private final Map deletions = new HashMap<>(); + + private void observe(String uri, FileState current, long now, long wallTime) { + FileState previous = observed.get(uri); + deletions.remove(uri); + if (current.sameVersion(previous)) { + stability.remove(uri); + return; + } + stability.observe( + new FileChangeEvent( + previous == null ? FileEventType.CREATED : FileEventType.MODIFIED, + current, + previous, + wallTime), + now); + } + + /** + * One event at a time: Hop rowsets supply downstream backpressure without another event queue. + */ + public FileChangeEvent next() throws IOException { + return next(clock.elapsedMillis(), clock.wallMillis()); + } + + public FileChangeEvent next(long now) throws IOException { + return next(now, now); + } + + private FileChangeEvent next(long now, long wallTime) throws IOException { + if (stopped) { + return null; + } + if (!deletions.isEmpty()) { + return deletions.values().iterator().next(); + } + int probes = Math.min(stability.size(), 1024); + if (now < nextProbe) { + return null; + } + for (int index = 0; index < probes; index++) { + String uri = stability.nextUri(); + // With stability disabled, a fresh stat is still performed immediately before emission. + if (!stability.due(uri, now, wallTime)) { + continue; + } + FileState current; + try { + long start = clock.elapsedMillis(); + current = scanner.readRelative(scanner.relativeUri(uri)); + if (diagnostics != null) diagnostics.probe(start); + } catch (IOException e) { + if (stopped) { + return null; + } + if (diagnostics != null) diagnostics.failure("stat"); + log.accept("Stability probe failed; pending state retained. Retrying: " + e.getMessage()); + nextProbe = now + retryInterval; + return null; + } + FileChangeEvent event = stability.check(uri, current, now, wallTime); + if (event != null) { + return event; + } + } + return null; + } + + /** + * Called only after putRow returned successfully (or a disabled event was intentionally + * observed). + */ + public void acknowledge(FileChangeEvent event) throws IOException { + acknowledge(event, true); + } + + public void acknowledge(FileChangeEvent event, boolean enabled) throws IOException { + String uri = event.file().getUri(); + if (event.getType() == FileEventType.DELETED) { + observed.remove(uri); + deletions.remove(uri); + } else { + if (!observed.containsKey(uri) && observed.size() >= maximumEntries) { + throw new IOException("Observed state entry limit exceeded. Increase maximum entries."); + } + observed.put(uri, event.getCurrent()); + stability.remove(uri); + } + dirty = true; + if (diagnostics != null) diagnostics.acknowledged(enabled); + report(false); + } + + public void await(long milliseconds) throws IOException, InterruptedException { + if (stopped) { + return; + } + // Retain the bounded batch for the next iteration; ordinary hints do not require a full scan. + FileWatcher.WatchBatch batch = watcher.poll(milliseconds); + if (!batch.getPaths().isEmpty() || batch.isReconciliationRequired()) { + deferredBatch = batch; + } + } + + private FileWatcher.WatchBatch deferredBatch; + + public void stop() throws IOException { + stopped = true; + watcher.close(); + } + + private Map snapshot() throws IOException { + long start = clock.elapsedMillis(); + Map result = scanner.snapshot(); + if (diagnostics != null) diagnostics.scan(start); + return result; + } + + private void save() throws IOException { + long start = clock.elapsedMillis(); + try { + store.save(observed); + if (diagnostics != null) diagnostics.checkpoint(start); + } catch (IOException e) { + if (diagnostics != null) diagnostics.failure("checkpoint"); + throw e; + } + } + + private void report(boolean force) { + if (diagnostics != null) + diagnostics.report(observed.size(), stability.size() + deletions.size(), force); + } + + public void checkpoint() throws IOException { + checkpoint(clock.elapsedMillis()); + } + + public void checkpoint(long now) throws IOException { + if (dirty && now >= nextCheckpoint) { + save(); + dirty = false; + nextCheckpoint = now + checkpointInterval; + } + } + + @Override + public void close() throws IOException { + try { + stop(); + if (initialized && dirty) { + save(); + dirty = false; + } + } finally { + try { + report(true); + } finally { + // Remove diagnostics before releasing the OS lock, so an immediate restart cannot + // collide with the previous instance's registry entry. + if (diagnostics != null) diagnostics.unregister(); + store.close(); + } + } + } +} diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesMeta.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesMeta.java new file mode 100644 index 00000000000..4a45bd7e6de --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesMeta.java @@ -0,0 +1,662 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import java.util.List; +import java.util.UUID; +import lombok.Getter; +import lombok.Setter; +import org.apache.hop.core.CheckResult; +import org.apache.hop.core.ICheckResult; +import org.apache.hop.core.annotations.Transform; +import org.apache.hop.core.exception.HopTransformException; +import org.apache.hop.core.gui.plugin.GuiElementType; +import org.apache.hop.core.gui.plugin.GuiPlugin; +import org.apache.hop.core.gui.plugin.GuiWidgetElement; +import org.apache.hop.core.gui.plugin.GuiWidgetGroupType; +import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.row.IValueMeta; +import org.apache.hop.core.row.value.ValueMetaBoolean; +import org.apache.hop.core.row.value.ValueMetaDate; +import org.apache.hop.core.row.value.ValueMetaInteger; +import org.apache.hop.core.row.value.ValueMetaString; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.HopMetadataProperty; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.BaseTransformMeta; +import org.apache.hop.pipeline.transform.TransformMeta; + +@Getter +@Setter +@GuiPlugin +@Transform( + id = "WatchFiles", + name = "i18n::WatchFilesMeta.Name", + description = "i18n::WatchFilesMeta.Description", + image = "watchfiles.svg", + categoryDescription = "i18n:org.apache.hop.pipeline.transform:BaseTransform.Category.Input", + keywords = "i18n::WatchFilesMeta.Keywords", + supportedEngines = {"Local"}, + documentationUrl = "/pipeline/transforms/watchfiles.html") +public class WatchFilesMeta extends BaseTransformMeta { + public static final String GUI_PLUGIN_ELEMENT_PARENT_ID = "WATCH_FILES_OPTIONS"; + public static final String REPLAY_GUI_PARENT_ID = "WATCH_FILES_REPROCESS"; + + @HopMetadataProperty + @GuiWidgetElement( + id = "directory", + order = "0100", + type = GuiElementType.FOLDER, + label = "i18n::WatchFilesMeta.directory.Label", + toolTip = "i18n::WatchFilesMeta.directory.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.General", + groupOrder = "01") + private String directory; + + @HopMetadataProperty + @GuiWidgetElement( + id = "editWatchId", + order = "0150", + type = GuiElementType.CHECKBOX, + label = "i18n::WatchFilesMeta.editWatchId.Label", + toolTip = "i18n::WatchFilesMeta.editWatchId.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.Advanced", + groupOrder = "02") + private boolean editWatchId; + + @HopMetadataProperty + @GuiWidgetElement( + id = "watchId", + order = "0200", + type = GuiElementType.TEXT, + label = "i18n::WatchFilesMeta.watchId.Label", + toolTip = "i18n::WatchFilesMeta.watchId.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.Advanced", + groupOrder = "02") + private String watchId; + + @HopMetadataProperty + @GuiWidgetElement( + id = "patternSyntax", + order = "0250", + type = GuiElementType.COMBO, + label = "i18n::WatchFilesMeta.patternSyntax.Label", + toolTip = "i18n::WatchFilesMeta.patternSyntax.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.General", + groupOrder = "01", + comboValuesMethod = "patternSyntaxValues", + variables = false) + private String patternSyntax = "REGEXP"; + + @HopMetadataProperty + @GuiWidgetElement( + id = "includeWildcard", + order = "0300", + type = GuiElementType.TEXT, + label = "i18n::WatchFilesMeta.includeWildcard.Label", + toolTip = "i18n::WatchFilesMeta.includeWildcard.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.General", + groupOrder = "01") + private String includeWildcard; + + @HopMetadataProperty + @GuiWidgetElement( + id = "excludeWildcard", + order = "0400", + type = GuiElementType.TEXT, + label = "i18n::WatchFilesMeta.excludeWildcard.Label", + toolTip = "i18n::WatchFilesMeta.excludeWildcard.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.General", + groupOrder = "01") + private String excludeWildcard; + + @HopMetadataProperty + @GuiWidgetElement( + id = "includeSubdirectories", + order = "0500", + type = GuiElementType.CHECKBOX, + label = "i18n::WatchFilesMeta.includeSubdirectories.Label", + toolTip = "i18n::WatchFilesMeta.includeSubdirectories.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.General", + groupOrder = "01") + private boolean includeSubdirectories = false; + + @HopMetadataProperty + @GuiWidgetElement( + id = "strategy", + order = "0600", + type = GuiElementType.COMBO, + label = "i18n::WatchFilesMeta.strategy.Label", + toolTip = "i18n::WatchFilesMeta.strategy.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.Advanced", + groupOrder = "02", + comboValuesMethod = "strategyValues", + variables = false) + private String strategy = "AUTO"; + + @HopMetadataProperty + @GuiWidgetElement( + id = "created", + order = "0700", + type = GuiElementType.CHECKBOX, + label = "i18n::WatchFilesMeta.created.Label", + toolTip = "i18n::WatchFilesMeta.created.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.General", + groupOrder = "01") + private boolean created = true; + + @HopMetadataProperty + @GuiWidgetElement( + id = "modified", + order = "0800", + type = GuiElementType.CHECKBOX, + label = "i18n::WatchFilesMeta.modified.Label", + toolTip = "i18n::WatchFilesMeta.modified.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.General", + groupOrder = "01") + private boolean modified = true; + + @HopMetadataProperty + @GuiWidgetElement( + id = "deleted", + order = "0900", + type = GuiElementType.CHECKBOX, + label = "i18n::WatchFilesMeta.deleted.Label", + toolTip = "i18n::WatchFilesMeta.deleted.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.General", + groupOrder = "01") + private boolean deleted = false; + + @HopMetadataProperty + @GuiWidgetElement( + id = "pollingInterval", + order = "1000", + type = GuiElementType.TEXT, + label = "i18n::WatchFilesMeta.pollingInterval.Label", + toolTip = "i18n::WatchFilesMeta.pollingInterval.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.Advanced", + groupOrder = "02") + private String pollingInterval = "5000"; + + @HopMetadataProperty + @GuiWidgetElement( + id = "reconciliationInterval", + order = "1100", + type = GuiElementType.TEXT, + label = "i18n::WatchFilesMeta.reconciliationInterval.Label", + toolTip = "i18n::WatchFilesMeta.reconciliationInterval.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.Advanced", + groupOrder = "02") + private String reconciliationInterval = "60000"; + + @HopMetadataProperty + @GuiWidgetElement( + id = "eventCapacity", + order = "1200", + type = GuiElementType.TEXT, + label = "i18n::WatchFilesMeta.eventCapacity.Label", + toolTip = "i18n::WatchFilesMeta.eventCapacity.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.Advanced", + groupOrder = "02") + private String eventCapacity = "4096"; + + @HopMetadataProperty + @GuiWidgetElement( + id = "maximumEntries", + order = "1300", + type = GuiElementType.TEXT, + label = "i18n::WatchFilesMeta.maximumEntries.Label", + toolTip = "i18n::WatchFilesMeta.maximumEntries.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.Advanced", + groupOrder = "02") + private String maximumEntries = "100000"; + + @HopMetadataProperty + @GuiWidgetElement( + id = "waitUntilStable", + order = "1400", + type = GuiElementType.CHECKBOX, + label = "i18n::WatchFilesMeta.waitUntilStable.Label", + toolTip = "i18n::WatchFilesMeta.waitUntilStable.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.General", + groupOrder = "01") + private boolean waitUntilStable = true; + + @HopMetadataProperty + @GuiWidgetElement( + id = "minimumAge", + order = "1500", + type = GuiElementType.TEXT, + label = "i18n::WatchFilesMeta.minimumAge.Label", + toolTip = "i18n::WatchFilesMeta.minimumAge.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.Advanced", + groupOrder = "02") + private String minimumAge = "2000"; + + @HopMetadataProperty + @GuiWidgetElement( + id = "stabilityChecks", + order = "1600", + type = GuiElementType.TEXT, + label = "i18n::WatchFilesMeta.stabilityChecks.Label", + toolTip = "i18n::WatchFilesMeta.stabilityChecks.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.Advanced", + groupOrder = "02") + private String stabilityChecks = "2"; + + @HopMetadataProperty + @GuiWidgetElement( + id = "stabilityInterval", + order = "1700", + type = GuiElementType.TEXT, + label = "i18n::WatchFilesMeta.stabilityInterval.Label", + toolTip = "i18n::WatchFilesMeta.stabilityInterval.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.Advanced", + groupOrder = "02") + private String stabilityInterval = "1000"; + + @HopMetadataProperty + @GuiWidgetElement( + id = "stateDirectory", + order = "1800", + type = GuiElementType.FOLDER, + label = "i18n::WatchFilesMeta.stateDirectory.Label", + toolTip = "i18n::WatchFilesMeta.stateDirectory.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.Advanced", + groupOrder = "02") + private String stateDirectory = ""; + + @HopMetadataProperty + @GuiWidgetElement( + id = "initialScan", + order = "1900", + type = GuiElementType.COMBO, + label = "i18n::WatchFilesMeta.initialScan.Label", + toolTip = "i18n::WatchFilesMeta.initialScan.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.General", + groupOrder = "01", + comboValuesMethod = "initialScanValues", + variables = false) + private String initialScan = "COMPARE_WITH_STATE"; + + @HopMetadataProperty + @GuiWidgetElement( + id = "checkpointInterval", + order = "2000", + type = GuiElementType.TEXT, + label = "i18n::WatchFilesMeta.checkpointInterval.Label", + toolTip = "i18n::WatchFilesMeta.checkpointInterval.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.Advanced", + groupOrder = "02") + private String checkpointInterval = "5000"; + + @HopMetadataProperty + @GuiWidgetElement( + id = "diagnosticsInterval", + order = "2100", + type = GuiElementType.TEXT, + label = "i18n::WatchFilesMeta.diagnosticsInterval.Label", + toolTip = "i18n::WatchFilesMeta.diagnosticsInterval.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.Advanced", + groupOrder = "02") + private String diagnosticsInterval = "60000"; + + @HopMetadataProperty + @GuiWidgetElement( + id = "slowOperationThreshold", + order = "2200", + type = GuiElementType.TEXT, + label = "i18n::WatchFilesMeta.slowOperationThreshold.Label", + toolTip = "i18n::WatchFilesMeta.slowOperationThreshold.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.Advanced", + groupOrder = "02") + private String slowOperationThreshold = "30000"; + + @HopMetadataProperty + @GuiWidgetElement( + id = "replayFilter", + order = "2300", + type = GuiElementType.TEXT, + label = "i18n::WatchFilesMeta.replayFilter.Label", + toolTip = "i18n::WatchFilesMeta.replayFilter.Tooltip", + parentId = REPLAY_GUI_PARENT_ID, + groupType = GuiWidgetGroupType.BOXES, + group = "i18n::WatchFilesMeta.Group.Reprocess", + groupOrder = "01") + private String replayFilter = ".*"; + + @HopMetadataProperty + @GuiWidgetElement( + id = "maximumRunTime", + order = "1950", + type = GuiElementType.TEXT, + label = "i18n::WatchFilesMeta.maximumRunTime.Label", + toolTip = "i18n::WatchFilesMeta.maximumRunTime.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.General", + groupOrder = "01") + private String maximumRunTime = ""; + + @HopMetadataProperty + @GuiWidgetElement( + id = "maximumRunTimeUnit", + order = "1960", + type = GuiElementType.COMBO, + label = "i18n::WatchFilesMeta.maximumRunTimeUnit.Label", + toolTip = "i18n::WatchFilesMeta.maximumRunTimeUnit.Tooltip", + parentId = GUI_PLUGIN_ELEMENT_PARENT_ID, + groupType = GuiWidgetGroupType.TABS, + group = "i18n::WatchFilesMeta.Group.General", + groupOrder = "01", + comboValuesMethod = "maximumRunTimeUnitValues", + variables = false) + private String maximumRunTimeUnit = "MINUTES"; + + public List maximumRunTimeUnitValues( + org.apache.hop.core.logging.ILogChannel log, IHopMetadataProvider provider) { + return guiValues("maximumRunTimeUnit"); + } + + public long maximumRunMillis(IVariables variables) { + String value = variables.resolve(maximumRunTime); + if (value == null || value.isBlank()) return 0; + try { + long multiplier = + switch (maximumRunTimeUnit) { + case "MINUTES" -> 60000L; + case "HOURS" -> 3600000L; + default -> throw new IllegalArgumentException("Unknown runtime unit."); + }; + long millis = + new java.math.BigDecimal(value.trim()) + .multiply(java.math.BigDecimal.valueOf(multiplier)) + .longValueExact(); + if (millis <= 0) throw new IllegalArgumentException("Non-positive runtime."); + return millis; + } catch (RuntimeException failure) { + throw new IllegalArgumentException( + "Maximum run time must be blank or a positive number in minutes or hours, representable in whole milliseconds.", + failure); + } + } + + public List strategyValues( + org.apache.hop.core.logging.ILogChannel log, IHopMetadataProvider provider) { + return guiValues("strategy"); + } + + public List patternSyntaxValues( + org.apache.hop.core.logging.ILogChannel log, IHopMetadataProvider provider) { + return guiValues("patternSyntax"); + } + + public String filenameRegex(IVariables variables, String value) { + return FilenamePatternSyntax.valueOf(patternSyntax).toRegex(variables.resolve(value)); + } + + public List initialScanValues( + org.apache.hop.core.logging.ILogChannel log, IHopMetadataProvider provider) { + return guiValues("initialScan"); + } + + private static List optionValues(String setting) { + return switch (setting) { + case "strategy" -> List.of("AUTO", "NATIVE", "POLLING"); + case "maximumRunTimeUnit" -> List.of("MINUTES", "HOURS"); + case "patternSyntax" -> List.of("WILDCARD", "REGEXP"); + case "initialScan" -> List.of("EMIT_EXISTING", "IGNORE_EXISTING", "COMPARE_WITH_STATE"); + default -> throw new IllegalArgumentException(setting); + }; + } + + private static List guiValues(String setting) { + return optionValues(setting).stream() + .map(value -> optionLabel(setting, value)) + .distinct() + .toList(); + } + + static String optionLabel(String setting, String value) { + if (value == null || !optionValues(setting).contains(value)) return value; + if ("initialScan".equals(setting) && "COMPARE_WITH_STATE".equals(value)) { + value = "IGNORE_EXISTING"; + } + return BaseMessages.getString(WatchFilesMeta.class, "WatchFilesMeta." + setting + "." + value); + } + + static String optionValue(String setting, String label) { + return optionValues(setting).stream() + .filter(value -> value.equals(label) || optionLabel(setting, value).equals(label)) + .findFirst() + .orElse(label); + } + + @Override + public boolean consumesMainInput() { + return false; + } + + @Override + public boolean canStartWithoutInput() { + return true; + } + + @Override + public PipelineMeta.PipelineType[] getSupportedPipelineTypes() { + return new PipelineMeta.PipelineType[] {PipelineMeta.PipelineType.Normal}; + } + + @Override + public void setDefault() { + directory = ""; + if (watchId == null || watchId.isBlank()) { + watchId = "watch-" + UUID.randomUUID(); + } + editWatchId = false; + patternSyntax = "WILDCARD"; + includeWildcard = ""; + excludeWildcard = ""; + includeSubdirectories = false; + strategy = "AUTO"; + created = true; + modified = true; + deleted = false; + pollingInterval = "5000"; + reconciliationInterval = "60000"; + eventCapacity = "4096"; + maximumEntries = "100000"; + waitUntilStable = true; + minimumAge = "2000"; + stabilityChecks = "2"; + stabilityInterval = "1000"; + stateDirectory = + java.nio.file.Path.of(System.getProperty("user.home"), ".hop", "watch-files").toString(); + initialScan = "COMPARE_WITH_STATE"; + checkpointInterval = "5000"; + diagnosticsInterval = "60000"; + slowOperationThreshold = "30000"; + replayFilter = ".*"; + maximumRunTime = ""; + maximumRunTimeUnit = "MINUTES"; + } + + public void validate(IVariables variables) { + if (variables.resolve(directory) == null || variables.resolve(directory).isBlank()) { + throw new IllegalArgumentException("Directory is required."); + } + String id = variables.resolve(watchId); + if (id == null || !id.matches("[A-Za-z0-9][A-Za-z0-9_-]{0,127}")) { + throw new IllegalArgumentException( + "Watch ID must contain 1-128 letters, digits, underscores or hyphens."); + } + if (variables.resolve(stateDirectory) == null || variables.resolve(stateDirectory).isBlank()) { + throw new IllegalArgumentException("A local state directory is required."); + } + DetectionStrategy.valueOf(strategy); + InitialScan.valueOf(initialScan); + maximumRunMillis(variables); + java.util.regex.Pattern.compile(filenameRegex(variables, includeWildcard)); + java.util.regex.Pattern.compile(filenameRegex(variables, excludeWildcard)); + number(variables, pollingInterval, 1, "Polling interval"); + number(variables, reconciliationInterval, 1, "Reconciliation interval"); + number(variables, checkpointInterval, 1, "Checkpoint interval"); + number(variables, diagnosticsInterval, 1, "Diagnostics interval"); + number(variables, slowOperationThreshold, 1, "Slow operation threshold"); + java.util.regex.Pattern.compile(variables.resolve(replayFilter)); + number(variables, minimumAge, 0, "Minimum age"); + number(variables, stabilityInterval, 1, "Stability interval"); + integer(variables, stabilityChecks, "Stability checks"); + integer(variables, eventCapacity, "Native hint capacity"); + integer(variables, maximumEntries, "Maximum entries"); + if (!created && !modified && !deleted) { + throw new IllegalArgumentException("Enable at least one event type."); + } + } + + public static long number(IVariables variables, String value, long minimum, String label) { + try { + long parsed = Long.parseLong(variables.resolve(value)); + if (parsed < minimum || parsed > Integer.MAX_VALUE) { + throw new NumberFormatException(); + } + return parsed; + } catch (RuntimeException e) { + throw new IllegalArgumentException( + label + " must be between " + minimum + " and " + Integer.MAX_VALUE + ".", e); + } + } + + public static int integer(IVariables variables, String value, String label) { + return (int) number(variables, value, 1, label); + } + + public boolean enabled(FileEventType type) { + return switch (type) { + case CREATED -> created; + case MODIFIED -> modified; + case DELETED -> deleted; + }; + } + + @Override + public void check( + List remarks, + PipelineMeta pipelineMeta, + TransformMeta transformMeta, + IRowMeta prev, + String[] input, + String[] output, + IRowMeta info, + IVariables variables, + IHopMetadataProvider metadataProvider) { + try { + validate(variables); + if (input.length != 0) { + throw new IllegalArgumentException( + "Watch Files is a source and does not accept input rows."); + } + remarks.add( + new CheckResult( + ICheckResult.TYPE_RESULT_OK, "Watch Files configuration is valid.", transformMeta)); + } catch (RuntimeException e) { + remarks.add(new CheckResult(ICheckResult.TYPE_RESULT_ERROR, e.getMessage(), transformMeta)); + } + } + + @Override + public void getFields( + IRowMeta row, + String name, + IRowMeta[] info, + TransformMeta nextTransform, + IVariables variables, + IHopMetadataProvider metadataProvider) + throws HopTransformException { + row.clear(); + IValueMeta[] fields = { + new ValueMetaString("filename"), + new ValueMetaString("short_filename"), + new ValueMetaString("path"), + new ValueMetaString("uri"), + new ValueMetaString("event_type"), + new ValueMetaInteger("size"), + new ValueMetaDate("last_modified"), + new ValueMetaDate("detected_at"), + new ValueMetaBoolean("is_directory"), + new ValueMetaString("scheme"), + new ValueMetaInteger("previous_size"), + new ValueMetaDate("previous_last_modified"), + new ValueMetaString("watch_id") + }; + for (IValueMeta field : fields) { + field.setOrigin(name); + row.addValueMeta(field); + } + } +} diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesReplayDialog.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesReplayDialog.java new file mode 100644 index 00000000000..75c96053e73 --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesReplayDialog.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import java.util.regex.Pattern; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.ui.core.PropsUi; +import org.apache.hop.ui.core.dialog.BaseDialog; +import org.apache.hop.ui.core.gui.GuiCompositeWidgets; +import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; +import org.eclipse.swt.SWT; +import org.eclipse.swt.layout.FormAttachment; +import org.eclipse.swt.layout.FormData; +import org.eclipse.swt.layout.FormLayout; +import org.eclipse.swt.widgets.Button; +import org.eclipse.swt.widgets.Dialog; +import org.eclipse.swt.widgets.Label; +import org.eclipse.swt.widgets.Shell; + +/** A one-time reprocessing selection; the main pipeline configuration is left unchanged. */ +class WatchFilesReplayDialog extends Dialog { + private final IVariables variables; + private final WatchFilesMeta options; + private boolean accepted; + private Label validation; + + WatchFilesReplayDialog(Shell parent, IVariables variables, WatchFilesMeta options) { + super(parent, BaseDialog.getDefaultDialogStyle()); + this.variables = variables; + this.options = options; + } + + boolean open() { + Shell shell = new Shell(getParent(), getStyle()); + shell.setText(message("ReprocessTitle")); + shell.setData(BaseDialog.DIALOG_SUBJECT, options); + PropsUi.setLook(shell); + FormLayout layout = new FormLayout(); + layout.marginWidth = 10; + layout.marginHeight = 10; + shell.setLayout(layout); + Button ok = new Button(shell, SWT.PUSH); + ok.setText(BaseMessages.getString(WatchFilesMeta.class, "System.Button.OK")); + Button cancel = new Button(shell, SWT.PUSH); + cancel.setText(BaseMessages.getString(WatchFilesMeta.class, "System.Button.Cancel")); + BaseTransformDialog.positionBottomButtons(shell, new Button[] {ok, cancel}, 10, null); + GuiCompositeWidgets widgets = + GuiCompositeWidgets.addScrolledComposite( + shell, + variables, + null, + ok, + WatchFilesMeta.REPLAY_GUI_PARENT_ID, + options, + groups -> + groups.registerExtraGroup( + BaseMessages.getString(WatchFilesMeta.class, "WatchFilesMeta.Group.Reprocess"), + "01", + null, + parent -> { + var existing = parent.getChildren(); + Label help = new Label(parent, SWT.WRAP); + help.setText(message("ReprocessHelp")); + FormData position = new FormData(); + position.left = new FormAttachment(0); + position.right = new FormAttachment(100); + position.top = new FormAttachment(existing[existing.length - 1], 10); + position.width = 570; + help.setLayoutData(position); + validation = new Label(parent, SWT.WRAP); + validation.setForeground( + parent.getDisplay().getSystemColor(SWT.COLOR_DARK_RED)); + FormData errorPosition = new FormData(); + errorPosition.left = new FormAttachment(0); + errorPosition.right = new FormAttachment(100); + errorPosition.top = new FormAttachment(help, 10); + errorPosition.width = 570; + errorPosition.height = 0; + validation.setLayoutData(errorPosition); + validation.setVisible(false); + errorPosition.top = new FormAttachment(existing[existing.length - 1], 10); + position.top = new FormAttachment(validation, 10); + })); + Runnable accept = + () -> { + widgets.getWidgetsContents(options, WatchFilesMeta.REPLAY_GUI_PARENT_ID); + try { + Pattern.compile(variables.resolve(options.getReplayFilter())); + accepted = true; + shell.dispose(); + } catch (RuntimeException invalid) { + validation.setText(message("InvalidReplayPattern") + invalid.getMessage()); + ((FormData) validation.getLayoutData()).height = SWT.DEFAULT; + validation.setVisible(true); + var content = validation.getParent(); + content.layout(true, true); + if (content.getParent() instanceof org.eclipse.swt.custom.ScrolledComposite scrolled) { + scrolled.setMinSize(content.computeSize(SWT.DEFAULT, SWT.DEFAULT)); + } + widgets.getWidgetsMap().get("replayFilter").setFocus(); + } + }; + ok.addListener(SWT.Selection, event -> accept.run()); + cancel.addListener(SWT.Selection, event -> shell.dispose()); + shell.setDefaultButton(ok); + shell.addListener( + SWT.Show, + event -> { + var screen = shell.getMonitor().getClientArea(); + var size = shell.getSize(); + shell.setSize(Math.min(size.x, screen.width), Math.min(size.y, screen.height)); + var point = shell.getLocation(); + size = shell.getSize(); + shell.setLocation( + Math.max(screen.x, Math.min(point.x, screen.x + screen.width - size.x)), + Math.max(screen.y, Math.min(point.y, screen.y + screen.height - size.y))); + }); + BaseDialog.defaultShellHandling(shell, value -> accept.run(), value -> shell.dispose()); + return accepted; + } + + private String message(String key) { + return BaseMessages.getString(WatchFilesMeta.class, "WatchFilesDialog." + key); + } +} diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesStateManager.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesStateManager.java new file mode 100644 index 00000000000..176de98c622 --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesStateManager.java @@ -0,0 +1,222 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.regex.Pattern; + +/** Local operator actions. Every mutation holds the same lifetime lock as the source. */ +public class WatchFilesStateManager { + private static final List SUFFIXES = + List.of(".json", ".json.tmp", ".json.bak", ".json.commit"); + private final Path directory; + private final String watchId; + private final int maximum; + + public WatchFilesStateManager(Path directory, String watchId, int maximum) throws IOException { + if (watchId == null || !watchId.matches("[A-Za-z0-9][A-Za-z0-9_-]{0,127}")) { + throw new IOException("Invalid Watch ID."); + } + if (maximum < 1) throw new IOException("Invalid maximum entries."); + this.directory = directory.toAbsolutePath().normalize(); + this.watchId = watchId; + this.maximum = maximum; + } + + public String inspect() throws IOException { + WatchFilesDiagnostics.Snapshot live = + WatchFilesDiagnostics.active(directory.resolve(watchId).toString()); + if (live != null) return "Active source\n" + live; + try (JsonFileStateStore store = open()) { + Map state = store.load(); + if (state == null) + return "No checkpoint. The next start follows First start without saved state."; + JsonNode header = new ObjectMapper().readTree(Files.readAllBytes(file(".json"))); + return "Stopped source\nWatch ID: " + + watchId + + "\nSchema: " + + header.path("version") + + "\nObserved files: " + + state.size() + + "\nSaved at (epoch ms): " + + header.path("savedAt") + + "\nGeneration: " + + header.path("generation") + + "\nRoot: " + + header.path("root").asText() + + "\nCheckpoint confirms source observation, not destination completion."; + } + } + + public Path backup() throws IOException { + try (JsonFileStateStore ignored = open()) { + return archive(directory, watchId); + } + } + + public int replay(String expression) throws IOException { + Pattern filter = Pattern.compile(expression); + try (JsonFileStateStore store = open()) { + Map state = store.load(); + if (state == null) throw new IOException("There is no checkpoint to replay."); + int before = state.size(); + state + .entrySet() + .removeIf(entry -> filter.matcher(entry.getValue().getShortFilename()).matches()); + int count = before - state.size(); + if (count != 0) { + archive(directory, watchId); + store.save(state); + } + return count; + } + } + + public Path reset() throws IOException { + try (JsonFileStateStore ignored = open()) { + Path history = archive(directory, watchId); + // Only this Watch ID's reserved files are affected. A backup must complete first. + for (String suffix : SUFFIXES) Files.deleteIfExists(file(suffix)); + forceDirectory(directory); + return history; + } + } + + public void migrate() throws IOException { + try (JsonFileStateStore store = open()) { + Map state = store.load(); + if (state == null) throw new IOException("There is no checkpoint to migrate."); + archive(directory, watchId); + store.save(state); + } + } + + public Path restoreBackup() throws IOException { + return restoreBackup(file(".json.bak")); + } + + public Path restoreBackup(Path source) throws IOException { + try (JsonFileStateStore ignored = open()) { + if (!Files.isRegularFile(source, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("No recovery backup (.json.bak) exists."); + } + // Validate the backup using the same reader in a unique scratch directory. Never overwrite + // an invalid main checkpoint merely to discover that its backup is also invalid. + Path scratch = Files.createTempDirectory(directory, watchId + ".restore-"); + try { + Files.copy(source, scratch.resolve(watchId + ".json")); + try (JsonFileStateStore candidate = + new JsonFileStateStore(scratch, watchId, null, null, maximum)) { + candidate.load(); + } + Path history = archive(directory, watchId); + copyForced(scratch.resolve(watchId + ".json"), file(".json.tmp")); + // Keep durable intent until replacement and forcing have completed, including on + // filesystems + // without atomic rename. A crash remains recoverable from the validated original backup. + try (FileChannel marker = + FileChannel.open( + file(".json.commit"), + StandardOpenOption.CREATE, + StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING)) { + marker.force(true); + } + forceDirectory(directory); + try { + Files.move( + file(".json.tmp"), + file(".json"), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING); + } catch (AtomicMoveNotSupportedException e) { + Files.move(file(".json.tmp"), file(".json"), StandardCopyOption.REPLACE_EXISTING); + } + force(file(".json")); + forceDirectory(directory); + Files.delete(file(".json.commit")); + forceDirectory(directory); + return history; + } finally { + Files.deleteIfExists(scratch.resolve(watchId + ".json")); + Files.deleteIfExists(scratch.resolve(watchId + ".lock")); + Files.delete(scratch); + } + } + } + + private JsonFileStateStore open() throws IOException { + return new JsonFileStateStore(directory, watchId, null, null, maximum); + } + + private Path file(String suffix) { + return directory.resolve(watchId + suffix); + } + + static Path archive(Path directory, String watchId) throws IOException { + Path parent = directory.resolve(watchId + ".history"); + if (Files.isSymbolicLink(parent)) throw new IOException("Backup directory must not be a link."); + Files.createDirectories(parent); + Path destination = + Files.createDirectory(parent.resolve(System.currentTimeMillis() + "-" + UUID.randomUUID())); + for (String suffix : SUFFIXES) { + Path original = directory.resolve(watchId + suffix); + if (Files.exists(original, LinkOption.NOFOLLOW_LINKS)) { + copyForced(original, destination.resolve(watchId + suffix)); + } + } + forceDirectory(destination); + forceDirectory(parent); + forceDirectory(directory); + return destination; + } + + private static void copyForced(Path source, Path target) throws IOException { + if (!Files.isRegularFile(source, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("State artifacts must be regular files."); + } + Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING); + force(target); + } + + private static void force(Path file) throws IOException { + try (FileChannel channel = FileChannel.open(file, StandardOpenOption.WRITE)) { + channel.force(true); + } + } + + private static void forceDirectory(Path directory) { + try (FileChannel channel = FileChannel.open(directory, StandardOpenOption.READ)) { + channel.force(true); + } catch (IOException | UnsupportedOperationException ignored) { + /* Best effort on Windows. */ + } + } +} diff --git a/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchLimitException.java b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchLimitException.java new file mode 100644 index 00000000000..3f41c5c435d --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/java/org/apache/hop/pipeline/transforms/watchfiles/WatchLimitException.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import java.io.IOException; + +/** A configured memory/resource limit is a configuration failure, not a transient listing error. */ +public class WatchLimitException extends IOException { + private static final long serialVersionUID = 1L; + + public WatchLimitException(String message) { + super(message); + } +} diff --git a/plugins/transforms/watchfiles/src/main/resources/org/apache/hop/pipeline/transforms/watchfiles/messages/messages_en_US.properties b/plugins/transforms/watchfiles/src/main/resources/org/apache/hop/pipeline/transforms/watchfiles/messages/messages_en_US.properties new file mode 100644 index 00000000000..a2db2dfab27 --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/resources/org/apache/hop/pipeline/transforms/watchfiles/messages/messages_en_US.properties @@ -0,0 +1,139 @@ +# /* +# * Licensed to the Apache Software Foundation (ASF) under one or more +# * contributor license agreements. See the NOTICE file distributed with +# * this work for additional information regarding copyright ownership. +# * The ASF licenses this file to You under the Apache License, Version 2.0 +# * (the "License"); you may not use this file except in compliance with +# * the License. You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# */ +# +WatchFilesMeta.Name=Watch Files +WatchFilesMeta.Description=Continuously emit stable file changes with persistent state and reconciliation +WatchFilesMeta.Keywords=watch,files,directory,monitor,source,vfs +WatchFilesDialog.Title=Watch Files +WatchFilesMeta.directory.Label=Directory +WatchFilesMeta.directory.Tooltip=Folder whose file changes should produce rows. Select a local folder or a named VFS connection.\nExample: a folder receiving CSV exports. Enable Include subdirectories to also watch its child folders. +WatchFilesMeta.editWatchId.Label=Edit state identifier +WatchFilesMeta.editWatchId.Tooltip=Show the saved identifier for manual editing. New transforms already have an automatically generated ID. Disabling this option only hides the field; it keeps the saved ID and history. Changing the identifier starts a separate history in the selected state directory. Existing checkpoints are preserved. +WatchFilesMeta.watchId.Label=State identifier +WatchFilesMeta.watchId.Tooltip=Generated once for a new transform and saved in the pipeline. Keep this ID to resume its checkpoint. Changing it starts a separate history; renaming the transform does not change it. Changing Directory, filters or Include subdirectories requires a new ID or an explicit state reset in Maintenance. +WatchFilesDialog.StateIdentity.Automatic=State identifier: Automatic +WatchFilesDialog.StateIdentity.Saved=State identifier: Saved +WatchFilesDialog.StateIdentityHelp=Saved identifier: {0}\nIt is reused on every start and is independent of the transform name and directory hash. To edit it, enable Edit state identifier in Advanced. Existing identifiers and checkpoints are retained. +WatchFilesMeta.patternSyntax.Label=Pattern syntax +WatchFilesMeta.patternSyntax.Tooltip=Choose how filename patterns are interpreted.\nWildcards: * matches any text, ? matches one character; examples: *.csv, report*, *test*. Other characters are literal.\nJava regular expression: use expressions such as .*\\.csv. Patterns match the whole filename and are case-sensitive. +WatchFilesMeta.includeWildcard.Label=Include filename pattern +WatchFilesMeta.includeWildcard.Tooltip=Match the entire filename using the syntax selected in General. Empty includes all files. Wildcards: * for all files, *.csv for CSV files, *test* for names containing test. Changing the matched scope requires a new Watch ID or a state reset in Maintenance. +WatchFilesMeta.excludeWildcard.Label=Exclude filename pattern +WatchFilesMeta.excludeWildcard.Tooltip=Match the entire filename using the syntax selected in General. Empty excludes nothing. Exclusion takes precedence. Changing the matched scope requires a new Watch ID or a state reset in Maintenance. +WatchFilesMeta.includeSubdirectories.Label=Include subdirectories +WatchFilesMeta.includeSubdirectories.Tooltip=Recursively monitor directories without following symbolic links. +WatchFilesMeta.strategy.Label=Detection strategy +WatchFilesMeta.strategy.Tooltip=Choose how folder changes are detected.\nAutomatic: use system notifications when available, otherwise scan periodically.\nNative: require system notifications and periodically verify the folder.\nPeriodic scans: repeatedly compare the folder with saved observations. +WatchFilesMeta.created.Label=Created +WatchFilesMeta.created.Tooltip=Produce a CREATED row when a matching new file is ready. Existing files also produce CREATED when Process existing files is selected for a first start without saved state. +WatchFilesMeta.modified.Label=Modified +WatchFilesMeta.modified.Tooltip=Produce a MODIFIED row when a previously observed file changes size or modification time and passes the readiness checks. Several writes can be combined into one stable change. +WatchFilesMeta.deleted.Label=Deleted +WatchFilesMeta.deleted.Tooltip=Produce a DELETED row when a previously observed file disappears. The row contains its last known metadata. Filter these rows before sending filenames to a content reader or copy operation. +WatchFilesMeta.pollingInterval.Label=Polling interval (ms) +WatchFilesMeta.pollingInterval.Tooltip=Time between complete folder scans when periodic detection is used. Also used to retry failed scans and metadata checks.\n5000 ms = 5 seconds. Shorter intervals increase scan frequency and filesystem load. Use a positive number or a variable such as '${SCAN_INTERVAL}'. +WatchFilesMeta.reconciliationInterval.Label=Reconciliation interval (ms) +WatchFilesMeta.reconciliationInterval.Tooltip=Time between complete verification scans when native notifications are used. These scans find persisting changes missed by notifications.\n60000 ms = 1 minute. Native events can be detected between scans. Shorter intervals increase filesystem load. +WatchFilesMeta.eventCapacity.Label=Native hint capacity +WatchFilesMeta.eventCapacity.Tooltip=Limit native notification hints handled per iteration. If there are too many, a complete verification scan is requested.\nDefault: 4096. Increase only when tuning a busy local folder. This option applies to native detection. +WatchFilesMeta.maximumEntries.Label=Maximum entries +WatchFilesMeta.maximumEntries.Tooltip=Limit how many files and directories this watcher can track.\nDefault: 100000. If the limit is exceeded, the watcher stops with an error rather than silently ignoring entries. Increase for larger folder trees, allowing for additional memory use. +WatchFilesMeta.waitUntilStable.Label=Wait until the file stops changing +WatchFilesMeta.waitUntilStable.Tooltip=Wait for repeated checks showing unchanged file size and modification time before producing CREATED or MODIFIED rows. Helps avoid processing files while they are being copied.\nTune the checks and spacing in Advanced. Minimum file age applies independently of this checkbox. +WatchFilesMeta.minimumAge.Label=Minimum file age (ms) +WatchFilesMeta.minimumAge.Tooltip=Minimum elapsed time since the file was last modified before it can be emitted. This applies even when Wait until the file stops changing is off.\n2000 ms = 2 seconds; 0 removes the age requirement. Variables are supported. +WatchFilesMeta.stabilityChecks.Label=Stability checks +WatchFilesMeta.stabilityChecks.Tooltip=Number of consecutive unchanged size and modification-time checks required after the first observation.\nDefault: 2. More checks provide a longer observation period; a detected change restarts the count. +WatchFilesMeta.stabilityInterval.Label=Stability interval (ms) +WatchFilesMeta.stabilityInterval.Tooltip=Minimum spacing between readiness checks.\n1000 ms = 1 second. Actual checks can occur later while scans or downstream processing are busy. Minimum file age is an additional requirement. Variables are supported. +WatchFilesMeta.stateDirectory.Label=State directory +WatchFilesMeta.stateDirectory.Tooltip=Local durable state directory. New transforms use .hop/watch-files under the user home directory. This folder must remain outside the watched tree and persist across executions. +WatchFilesMeta.initialScan.Label=First start without saved state +WatchFilesMeta.initialScan.Tooltip=Choose what to do only when this Watch ID has no saved state.\nProcess existing files: emit matching files already in the folder after readiness checks.\nWatch new changes only: record the current files as a quiet starting point.\nWith a valid saved checkpoint, always compare with it and resume observed changes. +WatchFilesMeta.checkpointInterval.Label=Checkpoint interval (ms) +WatchFilesMeta.checkpointInterval.Tooltip=Time between saving acknowledged file observations, also saved on graceful shutdown.\n5000 ms = 5 seconds. Shorter intervals write state more often. This records source observations, not successful downstream processing. + +WatchFilesMeta.diagnosticsInterval.Label=Diagnostics interval (ms) +WatchFilesMeta.diagnosticsInterval.Tooltip=How often to log counts and operation durations for monitoring and troubleshooting.\n60000 ms = 1 minute. A final summary is also logged on shutdown. Variables such as '${REPORT_INTERVAL}' are supported. +WatchFilesMeta.slowOperationThreshold.Label=Slow operation threshold (ms) +WatchFilesMeta.slowOperationThreshold.Tooltip=Log a warning when a completed folder scan, file check or state save takes at least this long.\n30000 ms = 30 seconds. Use this to spot slow storage or connections. Configure connection timeouts separately. +WatchFilesMeta.replayFilter.Label=Filename pattern +WatchFilesMeta.replayFilter.Tooltip=One-time selection for this reprocessing action. .* selects all names; .*\\.csv selects CSV files; .*test.* selects names containing test.\nUse a Java regular expression, independent of Pattern syntax in General. Only currently existing matching files can be emitted again after restart. +WatchFilesDialog.Inspect=View status +WatchFilesDialog.Backup=Create backup +WatchFilesDialog.Replay=Reprocess files... +WatchFilesDialog.Reset=Clear saved history... +WatchFilesDialog.Restore=Restore backup... +WatchFilesDialog.Migrate=Update saved-state format... +WatchFilesDialog.LocalStateRequired=State directory must be local. +WatchFilesDialog.BackupResult=Preserved checkpoint artifacts in:\u0020 +WatchFilesDialog.ReplayResult=Observed entries removed for replay on next start:\u0020 +WatchFilesDialog.StateError=State operation failed:\u0020 +WatchFilesDialog.ReplayConfirm=Re-emit currently existing files matching the replay filename regex on next start? This may duplicate destination data. The source must be stopped and CREATED events must be enabled. A backup is preserved first. +WatchFilesDialog.ResetConfirm=Back up and remove this Watch ID''s observed state? The next start follows First start without saved state. This can cause duplicates or establish a new quiet baseline. The source must be stopped. +WatchFilesDialog.RestoreConfirm=Restore the selected checkpoint after preserving current artifacts? It must belong to this Watch ID. An older checkpoint can cause duplicates. The source must be stopped. +WatchFilesDialog.MigrateConfirm=Preserve a backup and rewrite this checkpoint in the current schema? The source must be stopped. + +WatchFilesDialog.CreatedRequired=Enable CREATED events before replay. + +WatchFilesMeta.Group.General=General +WatchFilesMeta.Group.Advanced=Advanced +WatchFilesMeta.Group.Maintenance=Maintenance +WatchFilesMeta.strategy.AUTO=Automatic (recommended) +WatchFilesMeta.strategy.NATIVE=Native filesystem notifications +WatchFilesMeta.strategy.POLLING=Periodic folder scans +WatchFilesMeta.patternSyntax.WILDCARD=Wildcards (* and ?) +WatchFilesMeta.patternSyntax.REGEXP=Java regular expression +WatchFilesMeta.initialScan.EMIT_EXISTING=Process existing files +WatchFilesMeta.initialScan.IGNORE_EXISTING=Watch new changes only +WatchFilesMeta.initialScan.COMPARE_WITH_STATE=Resume saved state; otherwise watch new changes +WatchFilesDialog.GeneralHelp=Advanced contains detection, timing and state settings. A saved checkpoint is always resumed. Maintenance provides inspection and explicit reprocessing. +WatchFilesDialog.Patterns.WILDCARD=Filename patterns: * = all files; *.csv = CSV files; test* = starts with test; *test = ends with test; *test* = contains test. +WatchFilesDialog.Patterns.REGEXP=Filename patterns use Java regular expressions (selected in General): .* = all files; .*\\.csv = CSV files. The entire filename must match. + +WatchFilesDialog.Detection.AUTO=Automatic detection (recommended).\nLocal folders use system notifications when available; other locations or unsupported notifications use periodic scans.\nBoth intervals are shown because the detection method is chosen when the pipeline starts. Polling controls periodic scans and retries; reconciliation controls native verification scans. +WatchFilesDialog.Detection.NATIVE=Native filesystem notifications.\nReact to system file-change notifications and verify the folder periodically using Reconciliation interval. This mode requires a local filesystem with notification support; initialization fails if unavailable.\nMinimum file age and stability checks can delay emission until the file is ready. +WatchFilesDialog.Detection.POLLING=Periodic folder scans.\nRepeatedly list the folder and compare file size and modification time with saved observations. Polling interval controls the scan frequency.\nUseful for remote locations or network mounts with unreliable notifications. Shorter intervals increase filesystem load; readiness checks can add further delay. +WatchFilesDialog.PollingHelp.AUTO=Used when Automatic selects periodic scanning, and for retrying failed scans or file checks in either detection method.\n5000 ms = 5 seconds. A smaller value scans or retries more often and can increase filesystem load. Use a positive number or a variable such as '${SCAN_INTERVAL}'. +WatchFilesDialog.PollingHelp.POLLING=Time between complete folder scans. Each successful scan detects persisting creations, modifications and deletions relative to saved observations.\n5000 ms = 5 seconds. A smaller value scans more often and increases filesystem load. Busy scans, readiness checks or downstream processing can delay emission. Variables are supported. +WatchFilesDialog.PollingHelp.NATIVE=The saved polling interval is used to retry failed scans and file checks even in native mode.\n5000 ms = 5 seconds. Switch to Automatic to edit this retained setting. +WatchFilesDialog.ReconciliationHelp=Time between complete verification scans while using native notifications. These scans recover persisting changes missed by notifications.\n60000 ms = 1 minute. Native events can be detected between these scans. Smaller values increase scan frequency and filesystem load. Variables are supported. +WatchFilesDialog.IncludeHelp.WILDCARD=Select filenames to include with wildcard patterns.\n* or empty = all files; *.csv = CSV files; report* = starts with report; *test = ends with test; *test* = contains test; ? = one character.\nMatches the whole filename, including its extension, and is case-sensitive. Exclusions take precedence. Changing the matched scope requires a new Watch ID or a state reset in Maintenance. +WatchFilesDialog.IncludeHelp.REGEXP=Select filenames to include with a Java regular expression.\n.* or empty = all files; .*\\.csv = CSV files; .*test.* = contains test.\nMatches the whole filename and is case-sensitive. Exclusions take precedence. Changing the matched scope requires a new Watch ID or a state reset in Maintenance. +WatchFilesDialog.ExcludeHelp.WILDCARD=Exclude matching filenames even if the include pattern matches.\nEmpty = exclude nothing; *.tmp = temporary files; *backup* = names containing backup. * excludes all files.\nUses the wildcard syntax selected in General and is case-sensitive. Changing the matched scope requires a new Watch ID or a state reset in Maintenance. +WatchFilesDialog.ExcludeHelp.REGEXP=Exclude matching filenames even if the include pattern matches.\nEmpty = exclude nothing; .*\\.tmp = temporary files; .*backup.* = names containing backup. .* excludes all files.\nUses Java regular expressions and is case-sensitive. Changing the matched scope requires a new Watch ID or a state reset in Maintenance. + +WatchFilesMeta.Group.Reprocess=Files to reprocess +WatchFilesDialog.MaintenanceHelp=Saved history helps avoid re-emitting unchanged files.\nNo maintenance is needed for normal monitoring. +WatchFilesDialog.AdvancedRecovery=Advanced recovery +WatchFilesDialog.AdvancedRecoveryHelp=Show tools to clear saved history, restore a backup or update an older state format. Use these when repairing or restarting file tracking. +WatchFilesDialog.InspectHelp=Show saved observations or the running watcher diagnostics. Reading the status leaves the saved history unchanged. An observed file is not a confirmation of downstream processing. +WatchFilesDialog.BackupHelp=Stop the watcher, then create a backup of this Watch ID''s saved history. The current history and your monitored files remain unchanged. +WatchFilesDialog.ReplayHelp=Choose filenames to emit again after restarting the pipeline. Stop the watcher and enable Created first. You select the filter in a separate window and confirm before history is changed. Files can be processed more than once. +WatchFilesDialog.ResetHelp=Clear this Watch ID''s observed-file history after creating a backup. Monitored files are retained. The next start follows First start without saved state in General. A separate confirmation is required. +WatchFilesDialog.RestoreHelp=Select a backup to restore the watcher history. A backup of the current history is preserved first. Restoring an older history can cause files to be emitted again. Stop the watcher before confirming. +WatchFilesDialog.MigrateHelp=Convert saved history from an older state format to the current format, preserving a backup and the observed entries. This is an exceptional maintenance operation. Stop the watcher before confirming. +WatchFilesDialog.ReprocessTitle=Reprocess files +WatchFilesDialog.ReprocessHelp=Choose observed filenames to emit again on the next pipeline start.\nExamples: .* = all names; .*\\.csv = CSV files; .*test.* = names containing test.\nThis is a one-time regular expression, independent of Pattern syntax in General. Files must still exist, and Created must be enabled. A confirmation follows OK; Cancel leaves the history unchanged. +WatchFilesDialog.InvalidReplayTitle=Invalid filename pattern +WatchFilesDialog.InvalidReplayPattern=Enter a valid Java regular expression, for example .* or .*\\.csv.\n +WatchFilesMeta.maximumRunTime.Label=Maximum run time (blank = continuous) +WatchFilesMeta.maximumRunTime.Tooltip=Leave blank to watch continuously. Enter a positive duration, for example 30 minutes or 1.5 hours (use a decimal point), or a variable such as '${WATCH_DURATION}'. Timing starts when row processing begins, after initialization. At the limit, save state and finish normally so downstream rows can complete. A blocked filesystem operation or downstream handoff can delay completion. +WatchFilesMeta.maximumRunTimeUnit.Label=Run time unit +WatchFilesMeta.maximumRunTimeUnit.Tooltip=Choose minutes or hours for the maximum run time. For example, 90 minutes and 1.5 hours have the same duration. Shown only when a duration is entered. +WatchFilesMeta.maximumRunTimeUnit.MINUTES=Minutes +WatchFilesMeta.maximumRunTimeUnit.HOURS=Hours diff --git a/plugins/transforms/watchfiles/src/main/resources/version.xml b/plugins/transforms/watchfiles/src/main/resources/version.xml new file mode 100644 index 00000000000..36ab20e22eb --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/resources/version.xml @@ -0,0 +1,20 @@ + + + +${project.version} diff --git a/plugins/transforms/watchfiles/src/main/resources/watchfiles.svg b/plugins/transforms/watchfiles/src/main/resources/watchfiles.svg new file mode 100644 index 00000000000..6ef6002720e --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/resources/watchfiles.svg @@ -0,0 +1 @@ + diff --git a/plugins/transforms/watchfiles/src/main/samples/watch-files/input/README.md b/plugins/transforms/watchfiles/src/main/samples/watch-files/input/README.md new file mode 100644 index 00000000000..5d21eb1f811 --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/samples/watch-files/input/README.md @@ -0,0 +1,20 @@ + + +Place CSV files here while the Watch Files sample pipeline is running. +Write to a temporary non-CSV filename and rename when complete for the strongest producer protocol. +Stop the pipeline, add another CSV and restart to observe checkpoint recovery. diff --git a/plugins/transforms/watchfiles/src/main/samples/watch-files/watch-files.hpl b/plugins/transforms/watchfiles/src/main/samples/watch-files/watch-files.hpl new file mode 100644 index 00000000000..bf1348d4723 --- /dev/null +++ b/plugins/transforms/watchfiles/src/main/samples/watch-files/watch-files.hpl @@ -0,0 +1,60 @@ + + + + + watch-files + Observe stable CSV file changes, stop, and restart with persistent state. + + + + + Create or modify CSV files in ${PROJECT_HOME}/watch-files/input. Stop the pipeline to finish. Checkpoints live in watch-files/state. Restart after adding a file while stopped to observe recovery. + 404070080 + + + Watch directoryLog changesY + + Watch directoryWatchFiles1Y + ${PROJECT_HOME}/watch-files/input + sample-watch-files + REGEXP + .*\.csv + .*\.tmp + Y + AUTO + YYY + 5000 + 60000 + 4096100000 + Y2000 + 21000 + ${PROJECT_HOME}/watch-files/state + COMPARE_WITH_STATE5000 + 160180 + + + Log changesWriteToLog1Y + log_level_basicY + N0Stable filesystem change + + filenameevent_type + sizelast_modified + + 400180 + + diff --git a/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/FileReconcilerTest.java b/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/FileReconcilerTest.java new file mode 100644 index 00000000000..9cfec1a64c4 --- /dev/null +++ b/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/FileReconcilerTest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class FileReconcilerTest { + private FileState state(String uri, long size, long modified) { + return new FileState(uri, uri, uri, "/", "file", size, modified); + } + + @Test + void recoversMissedCreateModifyDeleteAndIgnoresUnchanged() { + FileState unchanged = state("a", 1, 1); + FileState previous = state("b", 2, 2); + FileState deleted = state("c", 3, 3); + FileState created = state("d", 4, 4); + FileState modified = state("b", 20, 2); + List changes = new ArrayList<>(); + new FileReconciler() + .compare( + Map.of("a", unchanged, "b", previous, "c", deleted), + Map.of("a", unchanged, "b", modified, "d", created), + 99, + changes::add); + assertEquals(3, changes.size()); + assertEquals(1, changes.stream().filter(e -> e.getType() == FileEventType.CREATED).count()); + FileChangeEvent modify = + changes.stream() + .filter(e -> e.getType() == FileEventType.MODIFIED) + .findFirst() + .orElseThrow(); + assertEquals(previous, modify.getPrevious()); + FileChangeEvent deletion = + changes.stream() + .filter(e -> e.getType() == FileEventType.DELETED) + .findFirst() + .orElseThrow(); + assertEquals(deleted, deletion.file()); + assertNull(deletion.getCurrent()); + assertEquals(99, deletion.getDetectedAt()); + } + + @Test + void detectsTimestampChangeWithoutSizeChange() { + List changes = new ArrayList<>(); + new FileReconciler() + .compare(Map.of("a", state("a", 1, 1)), Map.of("a", state("a", 1, 2)), 99, changes::add); + assertEquals(FileEventType.MODIFIED, changes.getFirst().getType()); + } +} diff --git a/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/FileStabilityTrackerTest.java b/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/FileStabilityTrackerTest.java new file mode 100644 index 00000000000..493df642dcf --- /dev/null +++ b/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/FileStabilityTrackerTest.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +class FileStabilityTrackerTest { + private FileState file(long size, long modified) { + return new FileState("a", "a", "a", "/", "file", size, modified); + } + + private FileChangeEvent created(FileState file) { + return new FileChangeEvent(FileEventType.CREATED, file, null, 0); + } + + @Test + void growingFileResetsConsecutiveChecksAndKeepsCreated() { + FileStabilityTracker tracker = new FileStabilityTracker(true, 0, 2, 100, 10); + FileState initial = file(10, 0); + tracker.observe(created(initial), 0); + assertNull(tracker.check("a", initial, 100)); + FileState growing = file(100, 200); + assertNull(tracker.check("a", growing, 200)); + assertNull(tracker.check("a", growing, 300)); + FileChangeEvent stable = tracker.check("a", growing, 400); + assertEquals(FileEventType.CREATED, stable.getType()); + assertEquals(100, stable.file().getSize()); + } + + @Test + void duplicateHintsDoNotRestartQuietInterval() { + FileStabilityTracker tracker = new FileStabilityTracker(true, 0, 2, 100, 10); + FileState file = file(10, 0); + tracker.observe(created(file), 0); + tracker.observe(created(file), 50); + assertNull(tracker.check("a", file, 100)); + tracker.observe(created(file), 150); + assertEquals(FileEventType.CREATED, tracker.check("a", file, 200).getType()); + } + + @Test + void minimumAgeIsRequiredEvenWithStabilityDisabled() { + FileStabilityTracker tracker = new FileStabilityTracker(false, 1000, 2, 100, 10); + FileState file = file(10, 100); + tracker.observe(created(file), 100); + assertNull(tracker.check("a", file, 1099)); + assertFalse(tracker.due("a", 1099)); + assertNotNull(tracker.check("a", file, 1100)); + } + + @Test + void observationsTooCloseDoNotCount() { + FileStabilityTracker tracker = new FileStabilityTracker(true, 0, 2, 100, 10); + FileState file = file(10, 0); + tracker.observe(created(file), 0); + assertNull(tracker.check("a", file, 99)); + assertNull(tracker.check("a", file, 100)); + assertNull(tracker.check("a", file, 199)); + assertNotNull(tracker.check("a", file, 200)); + } + + @Test + void disappearingUnemittedFileIsForgotten() { + FileStabilityTracker tracker = new FileStabilityTracker(true, 0, 2, 100, 10); + tracker.observe(created(file(10, 0)), 0); + assertNull(tracker.check("a", null, 100)); + assertEquals(0, tracker.size()); + } + + @Test + void capacityIsExplicitAndRepeatedModifiesStayOneCandidate() { + FileStabilityTracker tracker = new FileStabilityTracker(true, 0, 1, 100, 1); + FileState old = file(10, 0); + for (int i = 0; i < 100; i++) { + tracker.observe(new FileChangeEvent(FileEventType.MODIFIED, file(20 + i, i), old, i), i); + } + assertEquals(1, tracker.size()); + FileState second = new FileState("b", "b", "b", "/", "file", 0, 0); + assertThrows(IllegalStateException.class, () -> tracker.observe(created(second), 200)); + assertEquals(FileEventType.MODIFIED, tracker.check("a", file(119, 99), 200).getType()); + } +} diff --git a/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/FileStateStoreTest.java b/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/FileStateStoreTest.java new file mode 100644 index 00000000000..25783e84303 --- /dev/null +++ b/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/FileStateStoreTest.java @@ -0,0 +1,268 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.AccessDeniedException; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class FileStateStoreTest { + @TempDir Path directory; + + @Test + void missingMetadataMustNotSilentlyBecomeAnEmptyFileVersion() throws Exception { + try (JsonFileStateStore store = store()) { + store.save(files(10)); + } + Path state = directory.resolve("input.json"); + String invalid = Files.readString(state).replace("\"size\":10,", ""); + Files.writeString(state, invalid); + try (JsonFileStateStore store = store()) { + assertThrows(IOException.class, store::load); + assertThrows(IOException.class, () -> store.save(files(20))); + } + assertEquals(invalid, Files.readString(state)); + } + + private JsonFileStateStore store() throws IOException { + return new JsonFileStateStore(directory, "input", "root", "scope", 100); + } + + private Map files(long size) { + return Map.of("a", new FileState("a", "a", "a", "/", "file", size, 100)); + } + + @Test + void transientAccessDenialRetriesAtomicReplacementWithoutNonAtomicFallback() throws Exception { + AtomicInteger attempts = new AtomicInteger(); + try (JsonFileStateStore store = + new JsonFileStateStore(directory, "input", "root", "scope", 100) { + @Override + protected void atomicReplace() throws IOException { + if (attempts.incrementAndGet() < 3) { + throw new AccessDeniedException("state held open by a reader"); + } + super.atomicReplace(); + } + }) { + store.save(files(20)); + } + assertEquals(3, attempts.get()); + assertFalse(Files.exists(directory.resolve("input.json.commit"))); + try (JsonFileStateStore store = store()) { + assertEquals(files(20), store.load()); + } + } + + @Test + void persistentAccessDenialFailsAfterBoundedRetriesAndKeepsPreviousCheckpoint() throws Exception { + try (JsonFileStateStore store = store()) { + store.save(files(10)); + } + AtomicInteger attempts = new AtomicInteger(); + try (JsonFileStateStore store = + new JsonFileStateStore(directory, "input", "root", "scope", 100) { + @Override + protected void atomicReplace() throws IOException { + attempts.incrementAndGet(); + throw new AccessDeniedException("state is not replaceable"); + } + }) { + assertThrows(AccessDeniedException.class, () -> store.save(files(20))); + } + assertEquals(5, attempts.get()); + assertFalse(Files.exists(directory.resolve("input.json.commit"))); + try (JsonFileStateStore store = store()) { + assertEquals(files(10), store.load()); + } + } + + @Test + void interruptDuringRetryPreservesInterruptAndCheckpoint() throws Exception { + try (JsonFileStateStore store = store()) { + store.save(files(10)); + } + try (JsonFileStateStore store = + new JsonFileStateStore(directory, "input", "root", "scope", 100) { + @Override + protected void atomicReplace() throws IOException { + Thread.currentThread().interrupt(); + throw new AccessDeniedException("state is held open"); + } + }) { + try { + assertThrows(java.io.InterruptedIOException.class, () -> store.save(files(20))); + assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); + } + } + try (JsonFileStateStore store = store()) { + assertEquals(files(10), store.load()); + } + } + + @Test + void missingStateIsExplicitAndSaveReloadIsDurable() throws Exception { + try (JsonFileStateStore store = store()) { + assertNull(store.load()); + store.save(files(10)); + assertFalse(Files.exists(directory.resolve("input.json.tmp"))); + store.save(files(20)); + } + try (JsonFileStateStore store = store()) { + assertEquals(files(20), store.load()); + } + } + + @Test + void corruptStateIsNotOverwritten() throws Exception { + String broken = "{broken"; + Files.writeString(directory.resolve("input.json"), broken); + try (JsonFileStateStore store = store()) { + IOException error = assertThrows(IOException.class, store::load); + assertTrue(error.getMessage().contains("has not been overwritten")); + assertThrows(IOException.class, () -> store.save(files(20))); + } + assertEquals(broken, Files.readString(directory.resolve("input.json"))); + } + + @Test + void unsupportedVersionAndMismatchedScopeFail() throws Exception { + try (JsonFileStateStore store = store()) { + store.save(files(10)); + } + String saved = Files.readString(directory.resolve("input.json")); + Files.writeString( + directory.resolve("input.json"), saved.replace("\"version\":2", "\"version\":999")); + try (JsonFileStateStore store = store()) { + assertThrows(IOException.class, store::load); + } + Files.writeString(directory.resolve("input.json"), saved); + try (JsonFileStateStore store = + new JsonFileStateStore(directory, "input", "other", "scope", 100)) { + IOException error = assertThrows(IOException.class, store::load); + assertTrue(error.getMessage().contains("Directory differs")); + assertEquals(saved, Files.readString(directory.resolve("input.json"))); + } + try (JsonFileStateStore store = + new JsonFileStateStore(directory, "input", "root", "different", 100)) { + IOException error = assertThrows(IOException.class, store::load); + assertTrue(error.getMessage().contains("filters or Include subdirectories differ")); + assertTrue(error.getMessage().contains("new Watch ID")); + assertThrows(IOException.class, () -> store.save(files(20))); + assertEquals(saved, Files.readString(directory.resolve("input.json"))); + } + try (JsonFileStateStore store = + new JsonFileStateStore(directory, "all-files", "root", "different", 100)) { + assertNull(store.load()); + store.save(files(20)); + assertEquals(files(20), store.load()); + } + assertEquals(saved, Files.readString(directory.resolve("input.json"))); + } + + @Test + void simultaneousWritersAreRejectedAndLockCanBeReacquired() throws Exception { + try (JsonFileStateStore first = store()) { + assertThrows(IOException.class, this::store); + } + try (JsonFileStateStore second = store()) { + assertNull(second.load()); + } + } + + @Test + void nonWritableDirectoryFailsWithoutCreatingCheckpoint() throws Exception { + Path regularFile = Files.writeString(directory.resolve("not-a-directory"), "content"); + assertThrows( + IOException.class, + () -> new JsonFileStateStore(regularFile, "input", "root", "scope", 100)); + assertEquals("content", Files.readString(regularFile)); + } + + @Test + void failedAtomicReplacementPreservesPreviousCheckpoint() throws Exception { + try (JsonFileStateStore store = store()) { + store.save(files(10)); + } + try (JsonFileStateStore store = + new JsonFileStateStore(directory, "input", "root", "scope", 100) { + @Override + protected void atomicReplace() throws IOException { + throw new IOException("Injected access denial"); + } + }) { + assertThrows(IOException.class, () -> store.save(files(20))); + } + try (JsonFileStateStore store = store()) { + assertEquals(files(10), store.load()); + } + } + + @Test + void safeFallbackKeepsBackupAndReloadsReplacement() throws Exception { + try (JsonFileStateStore store = store()) { + store.save(files(10)); + } + try (JsonFileStateStore store = + new JsonFileStateStore(directory, "input", "root", "scope", 100) { + @Override + protected void atomicReplace() throws IOException { + throw new AtomicMoveNotSupportedException("tmp", "state", "Injected unsupported move"); + } + }) { + store.save(files(20)); + } + assertTrue(Files.exists(directory.resolve("input.json.bak"))); + assertFalse(Files.exists(directory.resolve("input.json.commit"))); + try (JsonFileStateStore store = store()) { + assertEquals(files(20), store.load()); + } + } + + @Test + void interruptedFallbackNeverStartsFreshOrOverwritesJournal() throws Exception { + Files.writeString(directory.resolve("input.json.commit"), "intent"); + try (JsonFileStateStore store = store()) { + assertThrows(IOException.class, store::load); + assertThrows(IOException.class, () -> store.save(files(20))); + } + assertEquals("intent", Files.readString(directory.resolve("input.json.commit"))); + assertFalse(Files.exists(directory.resolve("input.json"))); + } + + @Test + void invalidWatchIdCannotEscapeStateDirectory() { + assertThrows( + IOException.class, + () -> new JsonFileStateStore(directory, "../escape", "root", "scope", 100)); + } +} diff --git a/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/FilenamePatternSyntaxTest.java b/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/FilenamePatternSyntaxTest.java new file mode 100644 index 00000000000..90b32c2f1ab --- /dev/null +++ b/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/FilenamePatternSyntaxTest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.regex.Pattern; +import org.apache.hop.core.variables.Variables; +import org.apache.hop.core.xml.XmlHandler; +import org.apache.hop.metadata.serializer.memory.MemoryMetadataProvider; +import org.apache.hop.metadata.serializer.xml.XmlMetadataUtil; +import org.junit.jupiter.api.Test; + +class FilenamePatternSyntaxTest { + private boolean matches(String wildcard, String filename) { + return Pattern.compile(FilenamePatternSyntax.WILDCARD.toRegex(wildcard)) + .matcher(filename) + .matches(); + } + + @Test + void wildcardStarMatchesAllAndContainsPatternsMatchWholeFilenames() { + assertTrue(matches("*", "datos.csv")); + assertTrue(matches("*", "árbol\n😀.txt")); + assertTrue(matches("*test*", "pre-test-post.csv")); + assertTrue(matches("test*", "test.csv")); + assertTrue(matches("test*", "test")); + assertFalse(matches("test*", "pre-test.csv")); + assertTrue(matches("*test", "pre-test")); + assertTrue(matches("*test", "test")); + assertFalse(matches("*test", "test.csv")); + assertTrue(matches("*test.txt", "pre-test.txt")); + assertFalse(matches("*test.txt", "pre-test.txt.bak")); + assertFalse(matches("*test*", "data.csv")); + assertFalse(matches("*test*", "TEST.csv")); + assertTrue(matches("*.csv", "data.csv")); + assertFalse(matches("*.csv", "dataXcsv")); + } + + @Test + void wildcardQuestionMatchesOneCharacterAndRegexCharactersAreLiteral() { + assertTrue(matches("file?.txt", "file1.txt")); + assertTrue(matches("file?.txt", "file😀.txt")); + assertFalse(matches("file?.txt", "file12.txt")); + assertTrue(matches("file[1](a)+$.txt", "file[1](a)+$.txt")); + assertFalse(matches("file[1](a)+$.txt", "file1a.txt")); + assertTrue(matches("árbol😀.*", "árbol😀.csv")); + assertTrue(matches("\\E*", "\\Etest")); + } + + @Test + void regexpAndEmptyPatternsRetainTheirOriginalMeaning() { + assertEquals("", FilenamePatternSyntax.WILDCARD.toRegex(null)); + assertEquals("", FilenamePatternSyntax.WILDCARD.toRegex("")); + String expression = ".*\\.(csv|txt)"; + assertEquals(expression, FilenamePatternSyntax.REGEXP.toRegex(expression)); + assertThrows( + java.util.regex.PatternSyntaxException.class, + () -> Pattern.compile(FilenamePatternSyntax.REGEXP.toRegex("*"))); + } + + @Test + void oldXmlRetainsRegexpWhileNewGuiDefaultsUseWildcardsAndVariablesResolve() throws Exception { + WatchFilesMeta legacy = + XmlMetadataUtil.deSerializeFromXml( + XmlHandler.getSubNode( + XmlHandler.loadXmlString( + ".*\\.csv"), + "transform"), + WatchFilesMeta.class, + new MemoryMetadataProvider()); + Variables variables = new Variables(); + assertEquals("REGEXP", legacy.getPatternSyntax()); + assertEquals(".*\\.csv", legacy.filenameRegex(variables, legacy.getIncludeWildcard())); + WatchFilesMeta fresh = new WatchFilesMeta(); + fresh.setDefault(); + assertEquals("WILDCARD", fresh.getPatternSyntax()); + variables.setVariable("INCLUDE", "*test*"); + assertTrue( + Pattern.compile(fresh.filenameRegex(variables, "${INCLUDE}")) + .matcher("a-test.txt") + .matches()); + } +} diff --git a/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/LocalFileWatcherTest.java b/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/LocalFileWatcherTest.java new file mode 100644 index 00000000000..f6969830abb --- /dev/null +++ b/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/LocalFileWatcherTest.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardWatchEventKinds; +import java.nio.file.WatchEvent; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class LocalFileWatcherTest { + @TempDir Path root; + + private void awaitPath(LocalFileWatcher watcher, Path expected) throws Exception { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (System.nanoTime() < deadline) { + if (watcher.poll(100).getPaths().contains(expected)) { + return; + } + } + fail("No native event for " + expected); + } + + @Test + void detectCreateModifyDelete() throws Exception { + Path file = root.resolve("a.csv"); + try (LocalFileWatcher watcher = new LocalFileWatcher(root, false, 100, 100)) { + Files.writeString(file, "a"); + awaitPath(watcher, file); + Files.writeString(file, "longer"); + awaitPath(watcher, file); + Files.delete(file); + awaitPath(watcher, file); + } + } + + @Test + void existingAndNewSubdirectoriesAreRegistered() throws Exception { + Path existing = Files.createDirectories(root.resolve("existing/deep")); + try (LocalFileWatcher watcher = new LocalFileWatcher(root, true, 100, 100)) { + Path first = Files.writeString(existing.resolve("a.csv"), "a"); + awaitPath(watcher, first); + Path created = Files.createDirectories(root.resolve("new/deep")); + // A populated new subtree needs reconciliation as well as registration. + assertTrue(watcher.poll(1000).isReconciliationRequired()); + watcher.reconcileDirectories(); + Path second = Files.writeString(created.resolve("b.csv"), "b"); + awaitPath(watcher, second); + } + } + + @Test + void nonRecursiveWatcherDoesNotWatchChildren() throws Exception { + Path child = Files.createDirectory(root.resolve("child")); + try (LocalFileWatcher watcher = new LocalFileWatcher(root, false, 100, 100)) { + Files.writeString(child.resolve("a.csv"), "a"); + assertTrue(watcher.poll(200).getPaths().isEmpty()); + } + } + + @Test + void closeWakesBlockedPollAndCanBeRepeated() throws Exception { + LocalFileWatcher watcher = new LocalFileWatcher(root, false, 100, 100); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future blocked = executor.submit(() -> watcher.poll(60000)); + watcher.close(); + assertTrue(blocked.get(2, TimeUnit.SECONDS).getPaths().isEmpty()); + watcher.close(); + } finally { + watcher.close(); + executor.shutdownNow(); + assertTrue(executor.awaitTermination(2, TimeUnit.SECONDS)); + } + } + + @Test + void overflowExplicitlyRequestsReconciliation() throws Exception { + Path fakeRoot = mock(Path.class); + FileSystem fileSystem = mock(FileSystem.class); + WatchService service = mock(WatchService.class); + WatchKey key = mock(WatchKey.class); + when(fakeRoot.getFileSystem()).thenReturn(fileSystem); + when(fileSystem.newWatchService()).thenReturn(service); + when(fakeRoot.register( + service, + StandardWatchEventKinds.ENTRY_CREATE, + StandardWatchEventKinds.ENTRY_MODIFY, + StandardWatchEventKinds.ENTRY_DELETE)) + .thenReturn(key); + WatchEvent overflow = mock(WatchEvent.class); + doReturn(StandardWatchEventKinds.OVERFLOW).when(overflow).kind(); + doReturn(List.of(overflow)).when(key).pollEvents(); + when(key.reset()).thenReturn(true); + when(service.poll(0, TimeUnit.MILLISECONDS)).thenReturn(key); + try (LocalFileWatcher watcher = new LocalFileWatcher(fakeRoot, false, 2, 10)) { + FileWatcher.WatchBatch batch = watcher.poll(0); + assertTrue(batch.isReconciliationRequired()); + assertEquals(1, batch.getOverflowCount()); + } + verify(service).close(); + } + + @Test + void hintCapacityRequestsRecoveryInsteadOfUnboundedMemory() throws Exception { + try (LocalFileWatcher watcher = new LocalFileWatcher(root, false, 2, 10)) { + for (int i = 0; i < 20; i++) { + Files.writeString(root.resolve(i + ".csv"), "a"); + } + FileWatcher.WatchBatch batch = watcher.poll(1000); + assertTrue(batch.getPaths().size() <= 2); + assertTrue(batch.isReconciliationRequired()); + } + } + + @Test + void cappedDrainRemovesInvalidKeyFromBothRegistriesAndAllowsRegistrationAgain() throws Exception { + Path fakeRoot = mock(Path.class); + Path child = mock(Path.class); + FileSystem fileSystem = mock(FileSystem.class); + WatchService service = mock(WatchService.class); + WatchKey first = mock(WatchKey.class); + WatchKey invalid = mock(WatchKey.class); + WatchKey replacement = mock(WatchKey.class); + when(fakeRoot.getFileSystem()).thenReturn(fileSystem); + when(fileSystem.newWatchService()).thenReturn(service); + when(fakeRoot.register( + service, + StandardWatchEventKinds.ENTRY_CREATE, + StandardWatchEventKinds.ENTRY_MODIFY, + StandardWatchEventKinds.ENTRY_DELETE)) + .thenReturn(first); + when(child.register( + service, + StandardWatchEventKinds.ENTRY_CREATE, + StandardWatchEventKinds.ENTRY_MODIFY, + StandardWatchEventKinds.ENTRY_DELETE)) + .thenReturn(invalid, replacement); + when(first.reset()).thenReturn(true); + when(invalid.reset()).thenReturn(false); + when(service.poll(0, TimeUnit.MILLISECONDS)).thenReturn(first); + when(service.poll()).thenReturn(invalid, (WatchKey) null); + var register = LocalFileWatcher.class.getDeclaredMethod("register", Path.class); + register.setAccessible(true); + var keysField = LocalFileWatcher.class.getDeclaredField("keys"); + keysField.setAccessible(true); + var registeredField = LocalFileWatcher.class.getDeclaredField("registered"); + registeredField.setAccessible(true); + try (LocalFileWatcher watcher = new LocalFileWatcher(fakeRoot, false, 1, 2)) { + register.invoke(watcher, child); + java.util.Map keys = (java.util.Map) keysField.get(watcher); + java.util.Set registered = (java.util.Set) registeredField.get(watcher); + assertTrue(keys.containsKey(invalid)); + assertTrue(registered.contains(child)); + assertTrue(watcher.poll(0).isReconciliationRequired()); + verify(invalid).pollEvents(); + verify(invalid).reset(); + assertEquals(1, keys.size()); + assertFalse(keys.containsKey(invalid)); + assertFalse(registered.contains(child)); + register.invoke(watcher, child); + assertEquals(child, keys.get(replacement)); + assertTrue(registered.contains(child)); + assertEquals(2, keys.size()); + } + } + + @Test + void deletedDirectoryCanBeRegisteredAgainAndProducesRealNativeHints() throws Exception { + Path child = Files.createDirectory(root.resolve("child")); + try (LocalFileWatcher watcher = new LocalFileWatcher(root, true, 2, 2)) { + var field = LocalFileWatcher.class.getDeclaredField("registered"); + field.setAccessible(true); + java.util.Set registered = (java.util.Set) field.get(watcher); + assertTrue(registered.contains(child)); + Files.delete(child); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + boolean reconcile = false; + while (registered.contains(child) && System.nanoTime() < deadline) { + reconcile |= watcher.poll(100).isReconciliationRequired(); + } + assertTrue(reconcile); + assertFalse(registered.contains(child)); + Files.createDirectory(child); + deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + boolean recreated = false; + while (!recreated && System.nanoTime() < deadline) { + recreated = watcher.poll(100).isReconciliationRequired(); + } + assertTrue(recreated, "Expected the parent directory's creation notification"); + watcher.reconcileDirectories(); + assertTrue(registered.contains(child)); + Path file = Files.writeString(child.resolve("recreated.csv"), "first"); + awaitPath(watcher, file); + Files.writeString(file, "modified"); + awaitPath(watcher, file); + } + } +} diff --git a/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/VfsFileScannerTest.java b/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/VfsFileScannerTest.java new file mode 100644 index 00000000000..e552d612f12 --- /dev/null +++ b/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/VfsFileScannerTest.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.nio.file.DirectoryIteratorException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Iterator; +import org.apache.commons.vfs2.FileObject; +import org.apache.hop.core.HopEnvironment; +import org.apache.hop.core.variables.Variables; +import org.apache.hop.junit.rules.RestoreHopEngineEnvironmentExtension; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledOnOs; +import org.junit.jupiter.api.condition.OS; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +@ExtendWith(RestoreHopEngineEnvironmentExtension.class) +class VfsFileScannerTest { + @TempDir Path temporary; + + @BeforeAll + static void initializeHop() throws Exception { + HopEnvironment.init(); + } + + @ParameterizedTest + @ValueSource( + strings = { + "space name.csv", + "file#1.csv", + "file%20.csv", + "file%2520.csv", + "áéíóú-测试.csv", + "report[1].csv", + "report{1}+.csv" + }) + void portableNamesPreservePathsAndFileState(String filename) throws Exception { + assertName(filename); + } + + @EnabledOnOs(OS.LINUX) + @ParameterizedTest + @ValueSource( + strings = { + "report?.csv", + "ab.csv", + "a\"b.csv", + "a\\b.csv", + "a\nb.csv", + "a\tb.csv", + "a\u0001b.csv", + "a|b^`c.csv" + }) + void unixNamesPreservePathsAndFileState(String filename) throws Exception { + assertName(filename); + } + + private void assertName(String filename) throws Exception { + Path input = Files.createDirectory(temporary.resolve("input")); + Path file = Files.writeString(input.resolve(filename), "data"); + try (FileObject root = VfsFileScanner.resolveFile(input.toString(), new Variables())) { + VfsFileScanner scanner = new VfsFileScanner(root, true, "", "", 100, () -> false); + var snapshot = scanner.snapshot(); + assertEquals(1, snapshot.size()); + FileState state = snapshot.values().iterator().next(); + assertEquals(filename, state.getShortFilename()); + assertEquals(Files.size(file), state.getSize()); + assertEquals(Files.getLastModifiedTime(file).toMillis(), state.getLastModified()); + assertEquals("file", state.getScheme()); + String relative = scanner.relativeUri(state.getUri()); + assertEquals(state, scanner.readRelative(relative)); + assertEquals(state.getUri(), scanner.uri(relative)); + for (String location : + new String[] {file.toString(), file.toUri().toASCIIString(), state.getUri()}) { + try (FileObject resolved = VfsFileScanner.resolveFile(location, new Variables())) { + assertEquals(file.toAbsolutePath().normalize(), VfsFileScanner.localPath(resolved)); + assertTrue(resolved.exists()); + assertEquals(4, resolved.getContent().getSize()); + } + } + } + } + + @Test + void unusualRootAndNestedDirectoryPreserveRecursiveIdentity() throws Exception { + assertNested("root á%20 #[]{}+", "nested %2520 [1]"); + } + + @Test + @EnabledOnOs(OS.LINUX) + void unixRootAndNestedDirectoryPreserveRecursiveIdentity() throws Exception { + assertNested("root?\"<á%20>\\\n", "nested\\?\u0001"); + } + + private void assertNested(String rootName, String nestedName) throws Exception { + Path input = Files.createDirectory(temporary.resolve(rootName)); + Path file = + Files.writeString( + Files.createDirectory(input.resolve(nestedName)).resolve("data.csv"), "data"); + for (String location : new String[] {input.toString(), input.toUri().toASCIIString()}) { + try (FileObject root = VfsFileScanner.resolveFile(location, new Variables())) { + assertEquals(input, VfsFileScanner.localPath(root)); + VfsFileScanner scanner = new VfsFileScanner(root, true, "", "", 100, () -> false); + var snapshot = scanner.snapshot(); + assertEquals(1, snapshot.size()); + FileState state = snapshot.values().iterator().next(); + assertEquals(state, scanner.readRelative(scanner.relativeUri(state.getUri()))); + try (FileObject resolved = VfsFileScanner.resolveFile(state.getUri(), new Variables())) { + assertEquals(file, VfsFileScanner.localPath(resolved)); + } + } + } + } + + @Test + @EnabledOnOs(OS.WINDOWS) + void windowsDriveAndUncRootsArePreservedWithoutNetworkAccess() throws Exception { + for (String location : + new String[] { + "C:\\folder\\file%20.csv", "C:/folder/file%20.csv", "file:///C:/folder/file%2520.csv" + }) { + try (FileObject file = VfsFileScanner.resolveFile(location, new Variables())) { + assertEquals(Path.of("C:\\folder\\file%20.csv"), VfsFileScanner.localPath(file)); + } + } + for (String location : + new String[] { + "\\\\server\\share\\folder\\file%20.csv", + "file:////server/share/folder/file%2520.csv", + "file://server/share/folder/file%2520.csv" + }) { + try (FileObject file = VfsFileScanner.resolveFile(location, new Variables())) { + assertEquals( + Path.of("\\\\server\\share\\folder\\file%20.csv"), VfsFileScanner.localPath(file)); + } + } + } + + @Test + void variablesAreResolvedBeforeLocalPathConversion() throws Exception { + Variables variables = new Variables(); + variables.setVariable("INPUT", temporary.resolve("path %20 # á").toString()); + try (FileObject file = VfsFileScanner.resolveFile("${INPUT}", variables)) { + assertEquals(temporary.resolve("path %20 # á"), VfsFileScanner.localPath(file)); + } + } + + @Test + void localIterationIoFailureRemainsRetryableAndPreservesCloseFailure() throws Exception { + Path input = Files.createDirectory(temporary.resolve("input")); + IOException original = new IOException("Listing interrupted"); + IOException closeFailure = new IOException("Stream close failed"); + @SuppressWarnings("unchecked") + DirectoryStream entries = mock(DirectoryStream.class); + @SuppressWarnings("unchecked") + Iterator iterator = mock(Iterator.class); + when(entries.iterator()).thenReturn(iterator); + when(iterator.hasNext()).thenThrow(new DirectoryIteratorException(original)); + doThrow(closeFailure).when(entries).close(); + try (FileObject root = VfsFileScanner.resolveFile(input.toString(), new Variables()); + var files = mockStatic(Files.class)) { + files.when(() -> Files.newDirectoryStream(input)).thenReturn(entries); + IOException failure = + assertThrows( + IOException.class, + () -> new VfsFileScanner(root, true, "", "", 100, () -> false).snapshot()); + assertSame(original, failure); + assertEquals(1, failure.getSuppressed().length); + assertSame(closeFailure, failure.getSuppressed()[0]); + verify(entries).close(); + } + } +} diff --git a/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesCrashTest.java b/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesCrashTest.java new file mode 100644 index 00000000000..52f48623951 --- /dev/null +++ b/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesCrashTest.java @@ -0,0 +1,244 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.pipeline.transforms.watchfiles; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.apache.commons.vfs2.FileObject; +import org.apache.hop.core.variables.Variables; +import org.apache.hop.core.vfs.HopVfs; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.io.TempDir; + +@Timeout(40) +class WatchFilesCrashTest { + @TempDir Path directory; + + static Map files(long size) { + return Map.of("a", new FileState("a", "a.csv", "a.csv", "/", "file", size, 100)); + } + + private JsonFileStateStore store() throws IOException { + return new JsonFileStateStore(directory, "input", "root", "scope", 100); + } + + private void killAt(String stage) throws Exception { + Path ready = directory.resolve("ready"); + Files.deleteIfExists(ready); + String classpath = + System.getProperty("surefire.test.class.path", System.getProperty("java.class.path")); + Process child = + new ProcessBuilder( + Path.of(System.getProperty("java.home"), "bin", "java").toString(), + "-cp", + classpath, + Worker.class.getName(), + directory.toString(), + stage) + .redirectErrorStream(true) + .redirectOutput(directory.resolve("child.log").toFile()) + .start(); + try { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(60); + while (!Files.exists(ready) && child.isAlive() && System.nanoTime() < deadline) + Thread.sleep(10); + assertTrue( + Files.exists(ready), + () -> { + try { + return Files.readString(directory.resolve("child.log")); + } catch (IOException e) { + return e.toString(); + } + }); + child.destroyForcibly(); + assertTrue(child.waitFor(5, TimeUnit.SECONDS)); + // Windows can signal process termination before its final kernel file handles are released. + // Retry only in this crash-restart fixture, never bypass a live owner's production lock. + long releaseDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (true) { + try (JsonFileStateStore ignored = + new JsonFileStateStore(directory, "input", null, null, 100)) { + break; + } catch (IOException e) { + if (System.nanoTime() >= releaseDeadline) throw e; + Thread.sleep(10); + } + } + } finally { + if (child.isAlive()) { + child.destroyForcibly(); + child.waitFor(5, TimeUnit.SECONDS); + } + } + } + + @Test + void killAfterTemporaryWriteKeepsPreviousCheckpointAndReleasesLock() throws Exception { + try (JsonFileStateStore store = store()) { + store.save(files(10)); + } + killAt("before-rename"); + try (JsonFileStateStore store = store()) { + assertEquals(files(10), store.load()); + } + } + + @Test + void killAfterAtomicReplacementRecoversNewCompleteCheckpoint() throws Exception { + try (JsonFileStateStore store = store()) { + store.save(files(10)); + } + killAt("after-rename"); + try (JsonFileStateStore store = store()) { + assertEquals(files(20), store.load()); + } + } + + @Test + void killDuringEmissionReplaysUnacknowledgedVersion() throws Exception { + emission("before-ack", true); + } + + @Test + void killAfterAcknowledgementBeforeCheckpointCanRepeatVersion() throws Exception { + emission("after-ack", true); + } + + @Test + void killAfterCheckpointDoesNotRepeatObservedVersion() throws Exception { + emission("after-checkpoint", false); + } + + @Test + void killAfterFallbackIntentRequiresExplicitRestore() throws Exception { + fallback("fallback-before-rename"); + } + + @Test + void killAfterNonAtomicReplacementRequiresExplicitRestore() throws Exception { + fallback("fallback-after-rename"); + } + + private void fallback(String stage) throws Exception { + try (JsonFileStateStore store = store()) { + store.save(files(10)); + } + killAt(stage); + try (JsonFileStateStore store = store()) { + assertThrows(IOException.class, store::load); + } + new WatchFilesStateManager(directory, "input", 100).restoreBackup(); + try (JsonFileStateStore store = store()) { + assertEquals(files(10), store.load()); + } + } + + private void emission(String stage, boolean repeat) throws Exception { + killAt(stage); + try (FileObject root = + HopVfs.getFileObject(directory.resolve("watch").toString(), new Variables()); + WatchFilesEngine engine = Worker.engine(directory, root)) { + engine.initialize(InitialScan.COMPARE_WITH_STATE); + assertEquals(repeat, engine.next() != null); + } + } + + /** Launched in another JVM; the parent forcibly terminates it at a deterministic boundary. */ + public static class Worker { + public static void main(String[] args) throws Exception { + Path directory = Path.of(args[0]); + String stage = args[1]; + if (stage.contains("rename")) { + try (JsonFileStateStore store = + new JsonFileStateStore(directory, "input", "root", "scope", 100) { + @Override + protected void atomicReplace() throws IOException { + if (stage.startsWith("fallback")) + throw new java.nio.file.AtomicMoveNotSupportedException( + "tmp", "state", "Injected fallback"); + if (stage.equals("before-rename")) pause(directory); + super.atomicReplace(); + if (stage.equals("after-rename")) pause(directory); + } + + @Override + protected void nonAtomicReplace() throws IOException { + if (stage.equals("fallback-before-rename")) pause(directory); + super.nonAtomicReplace(); + if (stage.equals("fallback-after-rename")) pause(directory); + } + }) { + store.save(files(20)); + } + } else { + Path watch = Files.createDirectories(directory.resolve("watch")); + try (FileObject root = HopVfs.getFileObject(watch.toString(), new Variables()); + WatchFilesEngine engine = engine(directory, root)) { + engine.initialize(InitialScan.IGNORE_EXISTING); + Files.writeString(watch.resolve("a.csv"), "event"); + FileChangeEvent event = null; + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (event == null && System.nanoTime() < deadline) { + engine.tick(); + event = engine.next(); + if (event == null) Thread.sleep(5); + } + if (event == null) throw new IOException("No event produced"); + if (stage.equals("before-ack")) pause(directory); + engine.acknowledge(event); + if (stage.equals("after-ack")) pause(directory); + engine.checkpoint(); + pause(directory); + } + } + } + + static WatchFilesEngine engine(Path directory, FileObject root) throws IOException { + return new WatchFilesEngine( + new VfsFileScanner(root, false, "", "", 100, () -> false), + new VfsPollingWatcher(), + new JsonFileStateStore(directory, "input", root.getName().getURI(), "scope", 100), + new FileStabilityTracker(false, 0, 1, 1, 100), + 1, + 1, + 1, + null, + 100, + message -> {}); + } + + static void pause(Path directory) throws IOException { + Files.writeString(directory.resolve("ready"), "ready"); + try { + new java.util.concurrent.CountDownLatch(1).await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IOException(e); + } + } + } +} diff --git a/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesDialogTest.java b/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesDialogTest.java new file mode 100644 index 00000000000..b3db7847fd5 --- /dev/null +++ b/plugins/transforms/watchfiles/src/test/java/org/apache/hop/pipeline/transforms/watchfiles/WatchFilesDialogTest.java @@ -0,0 +1,757 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.pipeline.transforms.watchfiles; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.lang.reflect.Field; +import java.util.ArrayList; +import java.util.List; +import org.apache.hop.core.annotations.Transform; +import org.apache.hop.core.gui.plugin.GuiRegistry; +import org.apache.hop.core.gui.plugin.GuiWidgetElement; +import org.apache.hop.core.plugins.PluginRegistry; +import org.apache.hop.core.plugins.TransformPluginType; +import org.apache.hop.core.variables.Variables; +import org.apache.hop.pipeline.PipelineMeta; +import org.apache.hop.pipeline.transform.TransformMeta; +import org.apache.hop.ui.testing.SwtBotTestBase; +import org.eclipse.swt.custom.CTabFolder; +import org.eclipse.swt.custom.ScrolledComposite; +import org.eclipse.swt.graphics.Point; +import org.eclipse.swt.graphics.Rectangle; +import org.eclipse.swt.widgets.Button; +import org.eclipse.swt.widgets.Composite; +import org.eclipse.swt.widgets.Control; +import org.eclipse.swtbot.swt.finder.utils.SWTUtils; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +/** Run with tools/with-isolated-display.sh and -Puitest. */ +@Tag("uitest") +class WatchFilesDialogTest extends SwtBotTestBase { + @BeforeAll + static void registerWidgetsAndTransform() throws Exception { + PluginRegistry.getInstance() + .registerPluginClass( + WatchFilesMeta.class.getClassLoader(), + new ArrayList<>(), + null, + WatchFilesMeta.class.getName(), + TransformPluginType.class, + Transform.class, + true); + GuiRegistry registry = GuiRegistry.getInstance(); + for (Field field : WatchFilesMeta.class.getDeclaredFields()) { + GuiWidgetElement element = field.getAnnotation(GuiWidgetElement.class); + if (element != null) { + var parent = registry.findGuiElements(WatchFilesMeta.class.getName(), element.parentId()); + if (parent == null || parent.findChild(element.id()) == null) { + registry.addGuiWidgetElement(WatchFilesMeta.class.getName(), element, field); + } + } + } + } + + private PipelineMeta pipeline(WatchFilesMeta meta) { + PipelineMeta pipeline = new PipelineMeta(); + pipeline.addTransform(new TransformMeta("WatchFiles", "Watch input", meta)); + return pipeline; + } + + @Test + void groupedTabsRemainAboveButtonsAtDefaultAndSmallSizes() { + WatchFilesMeta meta = new WatchFilesMeta(); + withDialog( + parent -> new WatchFilesDialog(parent, new Variables(), meta, pipeline(meta)).open(), + bot -> { + var dialog = bot.shell("Watch Files").activate(); + display.syncExec( + () -> { + CTabFolder folder = find(dialog.widget, CTabFolder.class); + assertNotNull(folder); + org.junit.jupiter.api.Assertions.assertArrayEquals( + new String[] {"Wildcards (* and ?)", "Java regular expression"}, + find( + (Composite) folder.getItem(0).getControl(), + org.eclipse.swt.widgets.Combo.class) + .getItems()); + org.junit.jupiter.api.Assertions.assertArrayEquals( + new String[] { + "Automatic (recommended)", + "Native filesystem notifications", + "Periodic folder scans" + }, + find( + (Composite) folder.getItem(1).getControl(), + org.eclipse.swt.widgets.Combo.class) + .getItems()); + ScrolledComposite viewport = find(dialog.widget, ScrolledComposite.class); + assertNotNull(viewport); + List tabs = new ArrayList<>(); + for (var item : folder.getItems()) { + tabs.add(item.getText()); + } + assertEquals(List.of("General", "Advanced", "Maintenance"), tabs); + Button ok = null; + for (Control control : dialog.widget.getChildren()) { + if (control instanceof Button button + && button + .getText() + .replace("&", "") + .equals(buttonLabel("System.Button.OK"))) { + ok = button; + } + } + assertNotNull(ok); + if (Boolean.getBoolean("watchfiles.gui.capture")) { + SWTUtils.captureScreenshot("target/watchfiles-dialog-default.png"); + } + for (Point size : List.of(dialog.widget.getSize(), new Point(560, 350))) { + dialog.widget.setSize(size); + dialog.widget.layout(true, true); + Rectangle bounds = displayBounds(viewport); + Rectangle buttons = displayBounds(ok); + assertFalse( + bounds.intersects(buttons), "Options viewport overlaps OK after resize"); + assertTrue(buttons.y >= bounds.y + bounds.height, "OK must remain below options"); + assertTrue(bounds.height > 50, "Options must remain usable"); + assertTrue(displayBounds(folder).intersection(bounds).height > 50); + if (viewport.getContent().getSize().x > viewport.getClientArea().width) { + assertTrue( + viewport.getHorizontalBar().getVisible(), + "Width overflow must be scrollable"); + assertTrue( + viewport.getHorizontalBar().getMaximum() + > viewport.getHorizontalBar().getThumb()); + } + if (viewport.getContent().getSize().y > viewport.getClientArea().height) { + // Packed content is clipped by the viewport; overflow must remain accessible. + assertTrue( + viewport.getVerticalBar().getVisible(), "Overflow must be scrollable"); + assertTrue( + viewport.getVerticalBar().getMaximum() + > viewport.getVerticalBar().getThumb()); + } + } + }); + if (Boolean.getBoolean("watchfiles.gui.capture")) { + // Let the native ConfigureNotify/redraw run before taking the optional QA artifact. + bot.sleep(250); + display.syncExec( + () -> SWTUtils.captureScreenshot("target/watchfiles-dialog-small.png")); + } + dialog.bot().button(buttonLabel("System.Button.Cancel")).click(); + }); + } + + @Test + void optionalRuntimeShowsUnitsAndPersistsFriendlyHourSelection() { + WatchFilesMeta meta = new WatchFilesMeta(); + meta.setDefault(); + withDialog( + parent -> new WatchFilesDialog(parent, new Variables(), meta, pipeline(meta)).open(), + bot -> { + var dialog = bot.shell("Watch Files").activate().bot(); + display.syncExec( + () -> + assertFalse( + findLabel(bot.shell("Watch Files").widget, "Run time unit").getVisible())); + dialog.textWithLabel("Maximum run time (blank = continuous)").setText("1.5"); + display.syncExec( + () -> + assertTrue( + findLabel(bot.shell("Watch Files").widget, "Run time unit").getVisible())); + dialog.comboBoxWithLabel("Run time unit").setSelection("Hours"); + dialog.textWithLabel("Maximum run time (blank = continuous)").setText(""); + display.syncExec( + () -> + assertFalse( + findLabel(bot.shell("Watch Files").widget, "Run time unit").getVisible())); + dialog.textWithLabel("Maximum run time (blank = continuous)").setText("${DURATION}"); + assertEquals("Hours", dialog.comboBoxWithLabel("Run time unit").getText()); + if (Boolean.getBoolean("watchfiles.gui.capture")) { + bot.sleep(250); + display.syncExec( + () -> SWTUtils.captureScreenshot("target/watchfiles-dialog-timeout.png")); + } + dialog.button(buttonLabel("System.Button.OK")).click(); + }); + assertEquals("${DURATION}", meta.getMaximumRunTime()); + assertEquals("HOURS", meta.getMaximumRunTimeUnit()); + withDialog( + parent -> new WatchFilesDialog(parent, new Variables(), meta, pipeline(meta)).open(), + bot -> { + var dialog = bot.shell("Watch Files").activate().bot(); + assertEquals("Hours", dialog.comboBoxWithLabel("Run time unit").getText()); + dialog.textWithLabel("Maximum run time (blank = continuous)").setText(""); + dialog.button(buttonLabel("System.Button.Cancel")).click(); + }); + assertEquals("${DURATION}", meta.getMaximumRunTime()); + assertEquals("HOURS", meta.getMaximumRunTimeUnit()); + } + + @Test + void okRoundTripsEveryConfiguredOption() { + WatchFilesMeta meta = new WatchFilesMeta(); + meta.setDirectory("${INPUT_FOLDER}"); + meta.setWatchId("test-watch"); + meta.setEditWatchId(true); + meta.setStateDirectory("${STATE_FOLDER}"); + meta.setPatternSyntax("WILDCARD"); + meta.setIncludeWildcard("*.csv"); + meta.setExcludeWildcard("ignore*"); + meta.setIncludeSubdirectories(true); + meta.setStrategy("POLLING"); + meta.setInitialScan("EMIT_EXISTING"); + meta.setCreated(false); + meta.setModified(false); + meta.setDeleted(true); + meta.setPollingInterval("1234"); + meta.setReconciliationInterval("4321"); + meta.setCheckpointInterval("5678"); + meta.setWaitUntilStable(false); + meta.setMinimumAge("9876"); + meta.setStabilityChecks("4"); + meta.setStabilityInterval("300"); + meta.setMaximumEntries("54321"); + meta.setEventCapacity("321"); + meta.setDiagnosticsInterval("2468"); + meta.setSlowOperationThreshold("1357"); + meta.setReplayFilter("recover.*"); + withDialog( + parent -> new WatchFilesDialog(parent, new Variables(), meta, pipeline(meta)).open(), + bot -> + bot.shell("Watch Files") + .activate() + .bot() + .button(buttonLabel("System.Button.OK")) + .click()); + assertEquals("${INPUT_FOLDER}", meta.getDirectory()); + assertEquals("test-watch", meta.getWatchId()); + assertTrue(meta.isEditWatchId()); + assertEquals("${STATE_FOLDER}", meta.getStateDirectory()); + assertEquals("WILDCARD", meta.getPatternSyntax()); + assertEquals("*.csv", meta.getIncludeWildcard()); + assertEquals("ignore*", meta.getExcludeWildcard()); + assertTrue(meta.isIncludeSubdirectories()); + assertEquals("POLLING", meta.getStrategy()); + assertEquals("EMIT_EXISTING", meta.getInitialScan()); + assertFalse(meta.isCreated()); + assertFalse(meta.isModified()); + assertTrue(meta.isDeleted()); + assertEquals("1234", meta.getPollingInterval()); + assertEquals("4321", meta.getReconciliationInterval()); + assertEquals("5678", meta.getCheckpointInterval()); + assertFalse(meta.isWaitUntilStable()); + assertEquals("9876", meta.getMinimumAge()); + assertEquals("4", meta.getStabilityChecks()); + assertEquals("300", meta.getStabilityInterval()); + assertEquals("54321", meta.getMaximumEntries()); + assertEquals("321", meta.getEventCapacity()); + assertEquals("2468", meta.getDiagnosticsInterval()); + assertEquals("1357", meta.getSlowOperationThreshold()); + assertEquals("recover.*", meta.getReplayFilter()); + } + + @Test + void friendlyChoicesAndConditionalFieldsPreserveIdentityAndHiddenValues() { + WatchFilesMeta meta = new WatchFilesMeta(); + meta.setDefault(); + String id = meta.getWatchId(); + withDialog( + parent -> new WatchFilesDialog(parent, new Variables(), meta, pipeline(meta)).open(), + bot -> { + var dialog = bot.shell("Watch Files").activate().bot(); + if (Boolean.getBoolean("watchfiles.gui.capture")) { + bot.sleep(250); + display.syncExec( + () -> SWTUtils.captureScreenshot("target/watchfiles-dialog-general.png")); + } + readinessCheckbox(bot.shell("Watch Files").widget).deselect(); + dialog + .comboBoxWithLabel("First start without saved state") + .setSelection("Process existing files"); + dialog.cTabItem("Advanced").activate(); + display.syncExec( + () -> { + var shell = bot.shell("Watch Files").widget; + assertTrue(findLabel(shell, "Polling interval (ms)").getVisible()); + assertTrue(findLabel(shell, "Reconciliation interval (ms)").getVisible()); + assertTrue( + findLabel(shell, "Detection strategy") + .getToolTipText() + .contains("Both intervals")); + }); + dialog + .comboBoxWithLabel("Detection strategy") + .setSelection("Native filesystem notifications"); + display.syncExec( + () -> { + var shell = bot.shell("Watch Files").widget; + assertFalse(findLabel(shell, "Polling interval (ms)").getVisible()); + assertTrue(findLabel(shell, "Reconciliation interval (ms)").getVisible()); + assertTrue(findLabel(shell, "Native hint capacity").getVisible()); + assertTrue( + findLabel(shell, "Detection strategy") + .getToolTipText() + .contains("Reconciliation interval")); + assertTrue( + findLabel(shell, "Reconciliation interval (ms)") + .getToolTipText() + .contains("1 minute")); + }); + dialog.comboBoxWithLabel("Detection strategy").setSelection("Periodic folder scans"); + display.syncExec( + () -> { + var shell = bot.shell("Watch Files").widget; + assertFalse(findLabel(shell, "Stability checks").getVisible()); + assertFalse(findLabel(shell, "Stability interval (ms)").getVisible()); + assertFalse(findLabel(shell, "Native hint capacity").getVisible()); + assertFalse(findLabel(shell, "Reconciliation interval (ms)").getVisible()); + assertTrue(findLabel(shell, "Minimum file age (ms)").getVisible()); + assertTrue(findLabel(shell, "Polling interval (ms)").getVisible()); + assertTrue( + findLabel(shell, "Polling interval (ms)") + .getToolTipText() + .contains("5 seconds")); + assertTrue( + findLabel(shell, "Detection strategy") + .getToolTipText() + .contains("Polling interval")); + }); + dialog.textWithLabel("Polling interval (ms)").setText("${POLL_INTERVAL}"); + dialog.comboBoxWithLabel("Detection strategy").setSelection("Automatic (recommended)"); + assertEquals("${POLL_INTERVAL}", dialog.textWithLabel("Polling interval (ms)").getText()); + assertEquals("60000", dialog.textWithLabel("Reconciliation interval (ms)").getText()); + dialog.comboBoxWithLabel("Detection strategy").setSelection("Periodic folder scans"); + if (Boolean.getBoolean("watchfiles.gui.capture")) { + bot.sleep(250); + display.syncExec( + () -> SWTUtils.captureScreenshot("target/watchfiles-dialog-advanced.png")); + } + dialog.cTabItem("General").activate(); + dialog.comboBoxWithLabel("Pattern syntax").setSelection("Java regular expression"); + display.syncExec( + () -> + assertTrue( + findLabel(bot.shell("Watch Files").widget, "Include filename pattern") + .getToolTipText() + .contains("Java regular expression"))); + dialog.comboBoxWithLabel("Pattern syntax").setSelection("Wildcards (* and ?)"); + display.syncExec( + () -> + assertTrue( + findLabel(bot.shell("Watch Files").widget, "Include filename pattern") + .getToolTipText() + .contains("*test*"))); + readinessCheckbox(bot.shell("Watch Files").widget).select(); + dialog.cTabItem("Advanced").activate(); + display.syncExec( + () -> + assertTrue( + findLabel(bot.shell("Watch Files").widget, "Stability checks").getVisible())); + dialog.button(buttonLabel("System.Button.OK")).click(); + }); + assertEquals(id, meta.getWatchId()); + assertEquals("POLLING", meta.getStrategy()); + assertEquals("EMIT_EXISTING", meta.getInitialScan()); + assertEquals("WILDCARD", meta.getPatternSyntax()); + assertEquals("${POLL_INTERVAL}", meta.getPollingInterval()); + assertEquals("4096", meta.getEventCapacity()); + assertEquals("60000", meta.getReconciliationInterval()); + } + + private org.eclipse.swtbot.swt.finder.widgets.SWTBotCheckBox readinessCheckbox(Composite shell) { + return checkboxWithLabel(shell, "Wait until the file stops changing"); + } + + private org.eclipse.swtbot.swt.finder.widgets.SWTBotCheckBox checkboxWithLabel( + Composite shell, String text) { + java.util.concurrent.atomic.AtomicReference