diff --git a/joom/DEPLOYMENT.md b/joom/DEPLOYMENT.md index 57eca825762..c845beb48bf 100644 --- a/joom/DEPLOYMENT.md +++ b/joom/DEPLOYMENT.md @@ -186,3 +186,29 @@ kubectl --context analytics-gke -n zeppelin-test scale deployment zeppelin-serve Старый релиз и его данные остаются на месте. Повторный `install-test-012.sh` возвращает pilot. PVC пилота помечены `helm.sh/resource-policy: keep`; их удаление требует отдельного решения после окончания проверки. + +## Optional Iron Head Spark SQL panel + +The notebook UI reads `/ai-config.json` once. A missing endpoint or +`{"enabled":false}` disables the integration. The Helm nginx gateway serves +`{"enabled":true,"panelUrl":"https://iron-head-stage.joom.ai/embed/sql"}` for +the pilot. No AI key, Zeppelin ticket or editor text is sent across the bridge. + +The independent panel generates SQL using published Spark agents. The host +adapter accepts messages only from its configured iframe origin/window and +checks notebook/context IDs. It inserts a new paragraph through the normal +notebook API (including server ACL checks), using the unique bound +`org.apache.zeppelin.spark.SparkSqlInterpreter` magic. When several Spark +settings are bound, an explicit notebook default must identify exactly one +of them; list order is never used to guess. Ambiguous bindings, +report/revision views and disconnected hosts permit copying only. A network +failure after submission has an unknown outcome: check the notebook before a +manual retry. Neither the panel nor the adapter runs the resulting paragraph. + +For UI-only changes, set `JAVA_HOME` and an explicit GAR `IMAGE` tag and run +`joom/build-ui-image.sh`. It builds the UI WAR and overlays it on the pinned, +validated server image in `joom/Dockerfile.ui`; interpreter and server JARs are +unchanged. The OCI revision records the UI source revision; `/api/version` +still identifies the underlying server build. Roll back by disabling the +runtime config or selecting the previous server image. The protocol and AI +backend remain independent when replacing this editor. diff --git a/joom/Dockerfile.ui b/joom/Dockerfile.ui new file mode 100644 index 00000000000..4cd7f2c120d --- /dev/null +++ b/joom/Dockerfile.ui @@ -0,0 +1,20 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# UI-only overlay: preserve the already validated server/interpreter runtime. +ARG SERVER_BASE=europe-west4-docker.pkg.dev/gcp-joomanalytics-platform/images/zeppelin@sha256:dab60e463b567291adfb73c1b8d7c79c4d97b81277488601c4bbe4a1f037db03 +FROM ${SERVER_BASE} +USER 0 +ARG SOURCE_REVISION +LABEL org.opencontainers.image.source="https://github.com/joomcode/zeppelin" \ + org.opencontainers.image.revision="${SOURCE_REVISION}" +RUN rm -f /opt/zeppelin/zeppelin-web-angular-*.war +COPY zeppelin-web-angular.war /opt/zeppelin/zeppelin-web-angular-0.12.1-joom-1.war +USER 1000:0 diff --git a/joom/PORTS.md b/joom/PORTS.md index 709039e5ef9..0a295f23380 100644 --- a/joom/PORTS.md +++ b/joom/PORTS.md @@ -1,3 +1,21 @@ + + # Joom changes on Apache Zeppelin 0.12.1 Base: upstream `v0.12.1`, commit `ae9cb72ff`. Fork version: `0.12.1-joom-1`. diff --git a/joom/build-ui-image.sh b/joom/build-ui-image.sh new file mode 100755 index 00000000000..1042fbf7b82 --- /dev/null +++ b/joom/build-ui-image.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail +cd "$(dirname "$0")/.." +: "${IMAGE:?Set IMAGE to the target server image tag}" +: "${JAVA_HOME:?Set JAVA_HOME to a JDK installation}" + +# Dependencies/SDK are the same as the regular joom/build.sh build. Keep the +# Docker context to one WAR; no notebook data or local configuration is sent. +if [[ "${SKIP_UI_BUILD:-0}" != "1" ]]; then + ( + cd zeppelin-web-angular + ./node/node ./node/node_modules/npm/bin/npm-cli.js run build + ) +fi +[[ -f zeppelin-web-angular/dist/zeppelin/index.html ]] +[[ -f zeppelin-web-angular/dist/zeppelin/WEB-INF/web.xml ]] +build_context="$(mktemp -d "${TMPDIR:-/tmp}/zeppelin-ui.XXXXXX")" +trap 'rm -rf "$build_context"' EXIT +"$JAVA_HOME/bin/jar" cf "$build_context/zeppelin-web-angular.war" -C zeppelin-web-angular/dist/zeppelin . +cp joom/Dockerfile.ui "$build_context/Dockerfile" +docker build --platform linux/amd64 --build-arg "SOURCE_REVISION=$(git rev-parse HEAD)" \ + --tag "$IMAGE" "$build_context" +docker push "$IMAGE" diff --git a/joom/test_ai_panel.cjs b/joom/test_ai_panel.cjs new file mode 100644 index 00000000000..369092343f8 --- /dev/null +++ b/joom/test_ai_panel.cjs @@ -0,0 +1,136 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Exercise the real host adapter without a browser or Zeppelin server. +const assert = require('node:assert/strict'); +const { test } = require('node:test'); +const fs = require('node:fs'); +const vm = require('node:vm'); +const crypto = require('node:crypto'); +const ts = require('../zeppelin-web-angular/node_modules/typescript'); +const source = fs.readFileSync(require('node:path').join(__dirname, + '../zeppelin-web-angular/src/app/pages/workspace/notebook/ai-panel/ai-panel.component.ts'), 'utf8'); +const compiled = ts.transpileModule(source, { compilerOptions: { + target: ts.ScriptTarget.ES2020, module: ts.ModuleKind.CommonJS, experimentalDecorators: true +}}).outputText; +const CHANNEL = { channel: 'joom-sql-draft', version: 1 }; +const binding = { name: 'prod', selected: true, interpreters: [ + { name: 'sql', class: 'org.apache.zeppelin.spark.SparkSqlInterpreter' } +] }; + +async function host(options = {}) { + const writes = [], replies = []; + const frame = { postMessage: (payload, origin) => replies.push({ payload, origin }) }; + const module = { exports: {} }; + const context = { + exports: module.exports, module, crypto: crypto.webcrypto, URL, AbortController, + window: { location: { origin: 'https://analytics.joom.ai' }, addEventListener() {}, removeEventListener() {}, + setTimeout, clearTimeout }, + fetch: async (url, request) => { + if (!request?.method) return { ok: options.config !== false, + json: async () => ({ enabled: true, panelUrl: 'https://iron-head-stage.joom.ai/embed/sql' }) }; + writes.push({ url, request }); + if (options.networkError) throw new Error('connection lost'); + return { ok: !options.denied, status: options.denied ? 403 : 200, + json: async () => ({ status: 'OK', body: 'new-paragraph' }) }; + }, + require: name => { + if (name === '@angular/core') return { Component: () => target => target, + Input: () => () => {}, ViewChild: () => () => {} }; + throw new Error(`Unexpected runtime dependency: ${name}`); + } + }; + vm.runInNewContext(compiled, context); + const panel = new module.exports.NotebookAiPanelComponent({ + getBase: () => 'https://analytics.joom.ai/balloon-gcp/', + skipTrailingSlash: value => value.replace(/\/$/, ''), + getRestApiBase: () => 'https://analytics.joom.ai/balloon-gcp/api' + }, { bypassSecurityTrustResourceUrl: value => value }, { markForCheck() {} }); + panel.noteId = 'note-a'; panel.connected = true; panel.bindings = [binding]; + panel.frame = { nativeElement: { contentWindow: frame } }; panel.open = true; + await panel.ngOnInit(); + panel.receive({ origin: 'https://iron-head-stage.joom.ai', source: frame, data: { ...CHANNEL, type: 'ready' } }); + const insert = (extra = {}) => panel.insert({ requestId: 'request-a', noteId: panel.noteId, + contextId: panel.contextId, dialect: 'spark', sql: 'SELECT 1', ...extra }); + return { panel, writes, replies, frame, insert }; +} + +test('missing optional configuration never interrupts a notebook', async () => { + const { panel, writes } = await host({ config: false }); + assert.equal(panel.enabled, false); assert.equal(writes.length, 0); +}); +test('unique bound Spark SQL magic is required', async () => { + const { panel } = await host(); + assert.equal(panel.sqlMagic, '%prod.sql'); + panel.bindings = [binding, { ...binding, name: 'other' }]; + assert.equal(panel.canInsert, false); +}); +test('an explicit notebook default disambiguates Spark and Comet; an unknown default does not', async () => { + const { panel } = await host(); + panel.bindings = [{ ...binding, id: 'prod-id' }, { ...binding, id: 'comet-id', name: 'comet' }]; + panel.defaultInterpreterGroup = 'prod-id'; + assert.equal(panel.sqlMagic, '%prod.sql'); + panel.defaultInterpreterGroup = 'comet'; + assert.equal(panel.sqlMagic, '%comet.sql'); + panel.defaultInterpreterGroup = 'unknown'; + assert.equal(panel.sqlMagic, null); +}); +test('untrusted origins and source windows cannot insert', async () => { + const { panel, frame, writes } = await host(); + const data = { ...CHANNEL, type: 'insert', requestId: 'bad', noteId: panel.noteId, + contextId: panel.contextId, dialect: 'spark', sql: 'SELECT 1' }; + panel.receive({ origin: 'https://evil.test', source: frame, data }); + panel.receive({ origin: 'https://iron-head-stage.joom.ai', source: {}, data }); + await new Promise(resolve => setImmediate(resolve)); + assert.equal(writes.length, 0); +}); +test('valid insertion writes a new paragraph exactly once', async () => { + const { insert, writes, replies } = await host(); + await Promise.all([insert(), insert()]); + assert.equal(writes.length, 1); + assert.match(writes[0].url, /notebook\/note-a\/paragraph$/); + assert.deepEqual(JSON.parse(writes[0].request.body), { title: 'AI SQL draft', text: '%prod.sql\nSELECT 1' }); + assert.equal(replies.at(-1).payload.ok, true); +}); +test('a late answer cannot enter another notebook', async () => { + const { panel, insert, writes } = await host(); + const contextId = panel.contextId; + panel.noteId = 'note-b'; panel.ngOnChanges({ noteId: {} }); + await insert({ noteId: 'note-a', contextId }); + assert.equal(writes.length, 0); +}); +test('offline and read-only hosts reject writes', async () => { + for (const field of ['connected', 'readOnly']) { + const { panel, insert, writes } = await host(); + panel[field] = field === 'readOnly'; + await insert(); assert.equal(writes.length, 0); + } +}); +test('server ACL denial is explained without a retry', async () => { + const { insert, writes, replies } = await host({ denied: true }); + await insert(); + assert.equal(writes.length, 1); + assert.equal(replies.at(-1).payload.ok, false); + assert.match(replies.at(-1).payload.error, /permission/); +}); +test('unknown network outcome is never automatically replayed', async () => { + const { insert, writes, replies } = await host({ networkError: true }); + await insert(); await insert(); + assert.equal(writes.length, 1); + assert.match(replies.at(-1).payload.error, /Check the notebook/); +}); + +test('an unknown embedded application stays hidden until the protocol handshake', async () => { + const { panel } = await host(); + panel.toggle(); panel.toggle(); + assert.equal(panel.frameReady, false); + assert.equal(panel.canInsert, false); + panel.ngOnDestroy(); +}); diff --git a/zeppelin-web-angular/src/app/pages/workspace/notebook/ai-panel/ai-panel.component.html b/zeppelin-web-angular/src/app/pages/workspace/notebook/ai-panel/ai-panel.component.html new file mode 100644 index 00000000000..66388646893 --- /dev/null +++ b/zeppelin-web-angular/src/app/pages/workspace/notebook/ai-panel/ai-panel.component.html @@ -0,0 +1,29 @@ + + diff --git a/zeppelin-web-angular/src/app/pages/workspace/notebook/ai-panel/ai-panel.component.less b/zeppelin-web-angular/src/app/pages/workspace/notebook/ai-panel/ai-panel.component.less new file mode 100644 index 00000000000..5f50d2d1d17 --- /dev/null +++ b/zeppelin-web-angular/src/app/pages/workspace/notebook/ai-panel/ai-panel.component.less @@ -0,0 +1,35 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +:host { + display: contents; +} +.ai-panel { + position: sticky; + top: 0; + align-self: flex-start; + padding: 8px; +} +.ai-panel.open { + flex: 0 0 460px; + max-width: 50vw; + height: 100vh; + border-left: 1px solid #ddd; + background: white; +} +iframe { + display: block; + width: 100%; + height: calc(100% - 45px); + border: 0; + margin-top: 8px; +} diff --git a/zeppelin-web-angular/src/app/pages/workspace/notebook/ai-panel/ai-panel.component.ts b/zeppelin-web-angular/src/app/pages/workspace/notebook/ai-panel/ai-panel.component.ts new file mode 100644 index 00000000000..1a9e82be7ba --- /dev/null +++ b/zeppelin-web-angular/src/app/pages/workspace/notebook/ai-panel/ai-panel.component.ts @@ -0,0 +1,238 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + ChangeDetectorRef, + Component, + ElementRef, + Input, + OnChanges, + OnDestroy, + OnInit, + ViewChild +} from '@angular/core'; +import { DomSanitizer, SafeResourceUrl } from '@angular/platform-browser'; +import { InterpreterBindingItem } from '@zeppelin/sdk'; +import { BaseUrlService } from '@zeppelin/services/base-url.service'; + +const CHANNEL = 'joom-sql-draft'; + +/** The host adapter knows editor state only; AI and credentials stay in Iron Head. */ +@Component({ + selector: 'zeppelin-notebook-ai-panel', + templateUrl: './ai-panel.component.html', + styleUrls: ['./ai-panel.component.less'] +}) +export class NotebookAiPanelComponent implements OnInit, OnChanges, OnDestroy { + @Input() noteId = ''; + @Input() readOnly = false; + @Input() connected = false; + @Input() bindings: InterpreterBindingItem[] = []; + @Input() defaultInterpreterGroup = ''; + @ViewChild('frame') frame?: ElementRef; + enabled = false; + open = false; + frameReady = false; + frameError = false; + private readyTimer?: number; + frameUrl?: SafeResourceUrl; + panelPageUrl = ''; + private panelOrigin = ''; + private contextId = this.newContextId(); + private insertionIds = new Set(); + private destroyed = false; + private receive = (event: MessageEvent) => { + if (!this.open || event.origin !== this.panelOrigin || event.source !== this.frame?.nativeElement.contentWindow) { + return; + } + const data = event.data; + if (!data || data.channel !== CHANNEL || data.version !== 1) { + return; + } + if (data.type === 'ready') { + this.frameReady = true; + this.frameError = false; + window.clearTimeout(this.readyTimer); + this.cdr.markForCheck(); + this.sendContext(); + } else if (data.type === 'insert') { + void this.insert(data); + } + }; + + get sqlMagic(): string | null { + const candidates: { magic: string; binding: InterpreterBindingItem }[] = []; + for (const binding of this.bindings) { + if (binding.selected) { + for (const item of binding.interpreters) { + if (item.class === 'org.apache.zeppelin.spark.SparkSqlInterpreter') { + candidates.push({ magic: `%${binding.name}.${item.name}`, binding }); + } + } + } + } + // Zeppelin 0.12 reports all available settings as bound. An explicit + // notebook default can disambiguate Spark vs. Comet without guessing from + // list order or reading paragraph contents. + const resolved = + candidates.length > 1 && this.defaultInterpreterGroup + ? candidates.filter( + ({ binding }) => + binding.id === this.defaultInterpreterGroup || binding.name === this.defaultInterpreterGroup + ) + : candidates; + return resolved.length === 1 ? resolved[0].magic : null; + } + + get canInsert(): boolean { + return this.frameReady && !!this.noteId && this.connected && !this.readOnly && !!this.sqlMagic; + } + + toggle() { + this.open = !this.open; + this.frameReady = false; + this.frameError = false; + window.clearTimeout(this.readyTimer); + if (this.open) { + this.readyTimer = window.setTimeout(() => { + this.frameError = !this.frameReady; + this.cdr.markForCheck(); + }, 15000); + } + } + + private newContextId(): string { + return Array.from(crypto.getRandomValues(new Uint32Array(4))).join('-'); + } + + async ngOnInit() { + window.addEventListener('message', this.receive); + try { + const base = this.baseUrl.skipTrailingSlash(this.baseUrl.getBase()); + const response = await fetch(`${base}/ai-config.json`, { credentials: 'same-origin', cache: 'no-store' }); + if (!response.ok) { + return; + } + const config = await response.json(); + if (this.destroyed || config.enabled !== true || typeof config.panelUrl !== 'string') { + return; + } + const url = new URL(config.panelUrl); + if (url.protocol !== 'https:' && !(url.protocol === 'http:' && url.hostname === 'localhost')) { + return; + } + url.searchParams.set('parentOrigin', window.location.origin); + this.panelOrigin = url.origin; + this.panelPageUrl = url.toString(); + this.frameUrl = this.sanitizer.bypassSecurityTrustResourceUrl(url.toString()); + this.enabled = true; + this.cdr.markForCheck(); + } catch { + // Optional configuration/AI failures must never interrupt the notebook. + } + } + + ngOnChanges(changes: { [key: string]: unknown }) { + if (changes.noteId) { + this.contextId = this.newContextId(); + this.insertionIds.clear(); + } + this.sendContext(); + } + + sendContext() { + this.post({ type: 'host-context', noteId: this.noteId, contextId: this.contextId, canInsert: this.canInsert }); + } + + private post(payload: object) { + if (!this.destroyed && this.open && this.panelOrigin) { + this.frame?.nativeElement.contentWindow?.postMessage( + { channel: CHANNEL, version: 1, ...payload }, + this.panelOrigin + ); + } + } + + // Messages carry IDs rather than editor text. Only an explicit Insert action writes a new paragraph. + private async insert(data: { + requestId?: string; + noteId?: string; + contextId?: string; + sql?: string; + dialect?: string; + }) { + const requestId = data.requestId; + if (typeof requestId !== 'string' || requestId.length > 100 || this.insertionIds.has(requestId)) { + return; + } + this.insertionIds.add(requestId); + const contextId = this.contextId; + const reply = (payload: object) => this.post({ type: 'insert-result', requestId, contextId, ...payload }); + if ( + !this.canInsert || + data.noteId !== this.noteId || + data.contextId !== contextId || + data.dialect !== 'spark' || + typeof data.sql !== 'string' || + !data.sql.trim() || + data.sql.length > 200000 + ) { + reply({ ok: false, error: 'Notebook, connection or Spark SQL binding changed. Copy SQL instead.' }); + return; + } + const controller = new AbortController(); + const timeout = window.setTimeout(() => controller.abort(), 20000); + try { + const response = await fetch( + `${this.baseUrl.getRestApiBase()}/notebook/${encodeURIComponent(this.noteId)}/paragraph`, + { + method: 'POST', + credentials: 'same-origin', + signal: controller.signal, + headers: { 'Content-Type': 'application/json', 'X-Requested-With': 'XMLHttpRequest' }, + body: JSON.stringify({ title: 'AI SQL draft', text: `${this.sqlMagic}\n${data.sql}` }) + } + ); + if (response.status === 401 || response.status === 403) { + reply({ ok: false, error: 'You do not have permission to add a paragraph. Sign in or copy SQL instead.' }); + return; + } + if (!response.ok) { + throw new Error('Insertion failed.'); + } + const result = await response.json(); + if (result.status !== 'OK') { + throw new Error('Insertion was not confirmed.'); + } + reply({ ok: true, paragraphId: result.body }); + } catch { + reply({ + ok: false, + error: 'Insertion was not confirmed. Check the notebook before retrying; it may have been saved.' + }); + } finally { + window.clearTimeout(timeout); + } + } + + ngOnDestroy() { + this.destroyed = true; + window.clearTimeout(this.readyTimer); + window.removeEventListener('message', this.receive); + } + + constructor( + private baseUrl: BaseUrlService, + private sanitizer: DomSanitizer, + private cdr: ChangeDetectorRef + ) {} +} diff --git a/zeppelin-web-angular/src/app/pages/workspace/notebook/notebook.component.html b/zeppelin-web-angular/src/app/pages/workspace/notebook/notebook.component.html index b58ac98ebf4..59226c75b37 100644 --- a/zeppelin-web-angular/src/app/pages/workspace/notebook/notebook.component.html +++ b/zeppelin-web-angular/src/app/pages/workspace/notebook/notebook.component.html @@ -99,5 +99,12 @@ + diff --git a/zeppelin-web-angular/src/app/pages/workspace/notebook/notebook.module.ts b/zeppelin-web-angular/src/app/pages/workspace/notebook/notebook.module.ts index 62899c39b5d..f8ea6e5756a 100644 --- a/zeppelin-web-angular/src/app/pages/workspace/notebook/notebook.module.ts +++ b/zeppelin-web-angular/src/app/pages/workspace/notebook/notebook.module.ts @@ -56,6 +56,7 @@ import { NotebookRoutingModule } from './notebook-routing.module'; import { NotebookComponent } from './notebook.component'; import { NotebookShareModule } from './share/share.module'; import { NotebookSidebarComponent } from './sidebar/sidebar.component'; +import { NotebookAiPanelComponent } from './ai-panel/ai-panel.component'; @NgModule({ declarations: [ @@ -71,7 +72,8 @@ import { NotebookSidebarComponent } from './sidebar/sidebar.component'; NotebookParagraphFooterComponent, NotebookParagraphControlComponent, NoteFormBlockComponent, - NotebookSidebarComponent + NotebookSidebarComponent, + NotebookAiPanelComponent ], imports: [ CommonModule,