From 494651e118f5d92738ea947fefe0a07bc5373082 Mon Sep 17 00:00:00 2001 From: Dymas Date: Thu, 29 Oct 2026 13:14:12 +0100 Subject: [PATCH] Release resources 0.3.0 with persistent Queue plugin and connector --- .gitignore | 4 + AGENTS.md | 12 ++ CHANGELOG.md | 7 + README.md | 27 +-- VERSION | 1 + archives/plugin-queues-0.1.0.zip | Bin 0 -> 8297 bytes catalog.json | 21 ++- docs/Home.md | 10 ++ docs/queues.md | 70 ++++++++ docs/releases.md | 9 + docs/resources.md | 26 +++ docs/starters.md | 13 ++ plugins/queues/README.md | 7 + plugins/queues/connector.py | 190 ++++++++++++++++++++ plugins/queues/plugin.json | 6 + plugins/queues/plugin.py | 68 +++++++ plugins/queues/static/queues.css | 23 +++ plugins/queues/static/queues.js | 15 ++ plugins/queues/templates/queues/detail.html | 26 +++ plugins/queues/templates/queues/index.html | 32 ++++ plugins/queues/tests/browser_check.py | 86 +++++++++ plugins/queues/tests/test_queue.py | 146 +++++++++++++++ tests/test_catalog.py | 50 ++++++ tools/build_catalog.py | 43 +++++ tools/sync_wikis.py | 95 ++++++++++ 25 files changed, 962 insertions(+), 25 deletions(-) create mode 100644 .gitignore create mode 100644 AGENTS.md create mode 100644 VERSION create mode 100644 archives/plugin-queues-0.1.0.zip create mode 100644 docs/Home.md create mode 100644 docs/queues.md create mode 100644 docs/releases.md create mode 100644 docs/resources.md create mode 100644 docs/starters.md create mode 100644 plugins/queues/README.md create mode 100644 plugins/queues/connector.py create mode 100644 plugins/queues/plugin.json create mode 100644 plugins/queues/plugin.py create mode 100644 plugins/queues/static/queues.css create mode 100644 plugins/queues/static/queues.js create mode 100644 plugins/queues/templates/queues/detail.html create mode 100644 plugins/queues/templates/queues/index.html create mode 100644 plugins/queues/tests/browser_check.py create mode 100644 plugins/queues/tests/test_queue.py create mode 100644 tests/test_catalog.py create mode 100644 tools/build_catalog.py create mode 100644 tools/sync_wikis.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..93c42f6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +__pycache__/ +*.py[cod] +.pytest_cache/ +.venv/ diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..c430b3b --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,12 @@ +- Always bump `VERSION` unless the only changed files are `Dockerfile`, `*.md`, or `VERSION` itself. +- Always document code and behavior changes in `CHANGELOG.md` unless the only changed files are `Dockerfile`, `*.md`, or `VERSION` itself. +- Update `README.md` whenever user-facing behavior, setup, runtime requirements, Docker usage, or provider state changes. +- Always commit and push changes with a clear commit message. +- when new wersion is published always build a new docker image and push to gitea repo +- always store newly developed plugins or themes in web-framework-resources repo, each plugin and theme will contain a json with metadata desciption. +- When needed, install project tools with `pip` or the platform package manager. +- LAN URLs and `github.com` URLs may be accessed when needed for this project. +- always cleanup after new image is pushed, keep only latest and 5 previouse image version delete older + +- Keep README.md concise. Update docs/ with detailed function and behavior changes and synchronize both Gitea wikis with `python tools/sync_wikis.py` before completing a release. +- Keep framework code in web-framework and new resource source in the adjacent web-framework-resources checkout. diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cd8314..3163ef9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +# 0.3.0 — 2026-10-29 + +- Add Queue 0.1.0 for framework 0.6.0+: durable FIFO job claims, encrypted payloads, renewable leases, progress, bounded job logs, cancellation, retries, and protected server-rendered views. +- Publish complete Queue source and archive with JSON metadata and a catalog checksum. +- Add a deterministic catalog builder preserving previous resource archives. +- Maintain a dedicated adjacent resource checkout, short README, detailed function wiki sources, and a two-repository wiki synchronizer. + # Starter plugins 0.2.0 — 2026-10-26 - Package Notes, Bookmarks, and Workspace status with their complete implementation, metadata, documentation, and namespaced page/widget templates in each plugin folder. diff --git a/README.md b/README.md index 6e9401e..39313c5 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,15 @@ # Web Framework Resources -Complete plugin packages and theme templates for [Web Framework](https://gitea.coreplay.eu/Dymas/web-framework). +Plugins and themes for [Web Framework](https://gitea.coreplay.eu/Dymas/web-framework). -Default catalog: `https://gitea.coreplay.eu/Dymas/web-framework-resources/raw/branch/main/catalog.json`. -Open **Config → Add resources**, search, and install a plugin or theme. Restart, then enable the plugin or select the theme. The initial packages are also bundled in the framework for offline first startup. +In **Config → Add resources**, install a resource, restart the app, then enable the plugin or select the theme. -## Folder layout +- Plugins: Notes, Bookmarks, Workspace status, and **Queue**. +- Themes: Default, Ocean, and Mint. +- Queue requires framework **0.6.0+**. It adds job progress, logs, cancellation, retries, and a Python connector for plugin workers. -Every resource has its own source folder: `plugins//` or `themes//`. Plugin v0.2.0 packages include `plugin.py`, `implementation.py`, `plugin.json`, README, and their page/widget templates. They depend on public framework APIs and the common shell, rather than on the shared starter implementation. Theme folders contain `theme.json` and any accompanying files; the bundled themes are presets using the framework design system. +Keep this checkout beside `web-framework/`. Each plugin/theme lives in its own folder with JSON metadata. Build archives and the catalog with `python tools/build_catalog.py`; preserve previous archives. -Every JSON manifest declares `id`, `name`, `description`, and `version`. Themes also declare `mode` (`dark`, `light`, `system`), `accent` (`violet`, `blue`, `green`), and `density` (`comfortable`, `compact`). Versioned ZIP archives contain the entire resource with files at the archive root. Catalog entries record type, metadata, relative archive path and SHA-256. +Detailed functions and publishing instructions: [Gitea wiki](https://gitea.coreplay.eu/Dymas/web-framework-resources/wiki) · [Local docs](docs/Home.md) · [Queue API](docs/queues.md). -## Persistence and updates - -Framework Compose mounts host folders `./plugins` and `./themes` outside the container, with one subfolder per resource. Image rebuilds and restarts preserve existing files. Missing defaults are copied locally from the image; no resource ZIPs are downloaded at startup. - -The app checks catalogs in the background on startup and through **Check for updates** on the configuration pages. Checks only compare metadata: they never replace code or download archives. Local versions use PEP 440 comparison; source provenance protects against same-ID packages from another publisher. Operators can disable startup checks with `[resources].check_updates_on_startup = false`. - -To update, stop the workspace, back up the resource folder, verify a newer archive against its catalog SHA-256, extract it into that folder, then restart. Keep the mounted resource folders and control-data volume. Saved plugin content and configuration remain in the control store. - -## Publish resources - -Store newly developed plugins and themes here with all required helpers, templates, and assets in their own subfolder. Increment the resource version, build its complete ZIP, and update `catalog.json` with schema version 1. Commit source, archive, and catalog together. Keep old versioned archives for existing installations. Publish catalogs and archives for unauthenticated read access; never include secrets. - -The initial six resources can be regenerated with `PYTHONPATH=. python tools/prepare_resources.py /path/to/web-framework-resources` from the framework repository. The exporter regenerates bundled source directories and their catalog; merge any custom entries before publishing. Only configure trusted repositories: plugin code runs with the application user's privileges after restart. +Update docs with code changes and run `python tools/sync_wikis.py` to publish both repositories' wikis. diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..0d91a54 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.3.0 diff --git a/archives/plugin-queues-0.1.0.zip b/archives/plugin-queues-0.1.0.zip new file mode 100644 index 0000000000000000000000000000000000000000..c786cd71d7b46a44a12153e93cba80fdea0152a4 GIT binary patch literal 8297 zcmaKS1yEhh*5$=r65QS0-Gc;zySsC7y;yJv?s{>8yA#|Y!4llv-JRk6^Vk2)%eydP|%nF0092o$N*8_Oe=XH0DysaLwNt9A}RJsUXmGTCLbqY zzbuOW$t61<0Qbb^Q6dw-(H9CZufu=|HAb5Kyx0@)e0*C|WV2ka(ecO|9+Qr2DVwOB z^ROXmO-?e1%pz0zK@oBBP3?0P76tsI$3DUBXH*Gi5hh*t01b#utW-j03%{$+Cue6l zJKx6GgRgoXsGc)SWAxKX^a#EfgLjaajVNZg=uw<003s#{1k5n3@qyw21jBLG(dEvU$$+)c}`m{gB*8Btno%t zhP@pf9hiHpiLEkh;NJlPJd8it0;9%QBaH@xFV#B{4bLC|=X~(iUu~Hi$k+$$fNJn6 z3(f8&<aP1*(gmP%*rt=paDa_B5 z*n0*V8YWLFf{1T^}E9qgJQcz)s0v7g)_T3L5*`r69 zh}Q@2x^Hlr`oKsT6ijww>VRwq9MZ&O<}=1T2T`P7@NupB*sr^TgAqQubt+iXLxmtr z^fe)Et^}GsrK$;moqNVy7pKz3C`^XRQ-nEZsZgJ;^Z6=JPRtBlFABu+S}`Ae zjfR5Coi9T(&4H3UU^j6i{NLJ*u|mHpj*yYu^6wX214*sE2;Q> za{bzF`wST0yh`-I!9eTfoL8uAxJsB2fChcQwbpWfi|R8tBR)jLn<5^d!D+nW-){_S zR;d9rXGxAbDYzJWTBJ&KNA=7s?!%#y5U+$q^_(-ldNHtL)?H&WhjHb)WPNKt({ zE%q0=WN*DMkGm5a{|@sgYaF$C<%x-=xW#`hAczt%RDVyN-R}ru7Z&1+Y*w8Kt(*li zS$GC%6O1xUXE5m)y>jpmZ<}O%_5xQkNX9Mm$Mkt)p+jS4>>n6bvg)+ex>^Wnjd@+P z@?a3PkW$I|&pK+YF2HH(V4(_(3d^=qv(D|A2mtWDn<4d*SN+0QMZH4)9LN3^P7+AJOiIrs^r6hRVoq7eSKm?&200B z2rvibgu#-lud)K?8o$w6(thTC5ARiY9D^>?!`2&?WTOjTm#m(%tvQaGFm2o#mPWTr zqi5%(e1;LY@M=p^p+FCSf4&T7XZg{II!S-%9@yhZ7)!lf$lN^t-3=#R$Yk3Kexo=s z(&M7Vg>b>&7+uVIG^x}5pi5yNMgx&`xwFedXRqLAvs=#N+FO6Y{86qyGEHCoXkJ1X z9&#Q>49n)9&s?2BgFE|-#66jYWiXZ(_n6GE4saNr3rAeIPX2v0u;{HH((EC^QS7)* zXj=pPf~5T?gB}rpcydN3@7n zU4j1B<+6FhOSak_<`wBPOE>Oqe~6ujpV zs&6%OY2KBcQC&Au=4q+ zyI9WRDPp7}6$nT+Y5##-!WK0!p1%AA219tX0a7}vhV-2K|#HQ-sSF*5b;vI+suIV!$|us zaF$U&pl;xNxnlIT;-!i4EVl%SlUk*)?X4YXMt9empp%BldopREvdACG7G?6p&p6^qG=WT#R4{XbTE1kS$3 z#F6hiZ?cA>bNJr0#MqF9Bc}9qK0UVglMb{;(2o(Zu53c1?^Ee5++Wha zNB^;`Qet`2TT}&~`4M>&`SJ5QE@k+r6iE?T+t1Aqq#7y3*ndO!nJ?%cP|O{wHtg8Hafc~FTCBN74(oWo6IZH&##Ah{ z;wUbJUF}FrevX#GH+taF7B^_uIINTbe=QXL;*RV*=t~V_=>%QeTxqg&(y!{SmZzw) zd0EQbCY#U!h8l-8Ih@mGl~*JdZ&z0SG*2?XeA=WvR3uTtkML59FWYfquDnW)cx$4# zkD6>+rDL3~0?LI=jnU za!QE7i=&SL{Z!`Q?WD6IyIN8^)+dhE`IzxKMMp=6umJX)8+=K`is69>;#}6l+i+$A zdL;OsVSLWY9EY!`AHqS9Demyy^=j4mdwPx?%gmAa9J=9WH-@flc=<2q^qV5iEmFV;%#2_4sD`3PwN5-L4l0?J>H|BNyL*4eCwg?#1Lj zI1#xgEO@ltPfwyGYeeTZ+HIEGV%xX+Vg~2SK(ad~)w$d&O0sH2Kh*-d4Ot_ht`BiQ++P9og5@u$KoC|I^FOo721% zef_QZu^{jwf@H{+OC3GBlT#nNeKnx8XkF#Ur%^6?Dc`u`+fr0gZT8yk26c47tOt#| z5wwl=x={R0eUFKNgh$YhkY>7nC^y-v1++dD4>x|}3lAS~VkjkHUBSeeu1=T5c&YGG zyvdIGi&B}2^Zd9k31h@hb?KeT%83q#ZLX=Z8<1_TlGXGHR2r^wV^Tqw^Ht~Eo-#8P zOD;%_6yJE{fdA1!Ofqj|H2RhpE}+-ouuJvcTXfx>8oeR-&{#BMLlHUv4x5Xz!h~R< z{6|tlhA^#Or{ZVd_>(Ku&dpH%xDUe*dIbk-BTB&d1`8zSGR6Y!Gg49EDTDhp^OC{IYK=6x0!&RfeAC0R9; zJnwuQ3F-nxE;}e5n=2HzS-YSaLxT^+$D3_+Uvz|7Xa$qJ9x0|esJ}Hxc=#ELPKwy) zB?!;tMJ}9L@+P0KC@bgioWtUw)7i$0-34vuZ}A`<*u5@TBKIWHV}(uJMSnCHk>5uS z0`YDtqd@cL3UCr#_jrR%TLeXDV$H6~f@HB*vk`R>`^rbjajb}4J1W>KDd4E}B@)jB zs#<1+g9T^)ymIl##2zLKy>gd+pAtX>P76JXY3o2T_pxv18{C4LYBsEp_RLBP4Z*c$4Uo>bAd_nKLrRd#|-p$d@&C=SQ z*~Z1eo@z`C6_O17)09wgNlkW~(-GY@JUlp^d)|m=zt__vsNOeu`i-~S<<|0kFIQ|g zmF^=6=L!Ew?w6D#-937j);N4!nu8i4dh_KYlsB~Y)-x*|9;$knBKsy%St-mKSCt5u z%BX0V(r-$K>V5HEV`~((p{iOPPi-XszQp9#Otvm406_Siz5a{9{`V3cy`nYbVwbql zJC8Jxee$O3>Y}MrPuQ)Hwtq|1iUk^A2Z(;ANRRnKPuciGYU+&``SU{E9IXZ`qwwyH zFUPi3qmFNeqd2a4EU6n3>Peyvx|D6UTI4N{C-*IZ4qT$dxd3#LkPZ`^C+{P8Y7)1y z_F^!le1`t0f=`r3<_=}rm~*x-r!TlbebFVZu{K+RLpyR&)0xmKZpCSz?-)E)Cg5B% zPahdw9B)6RL5#LVD)hVe4gtK!x{!Am75$oBTgA_SCR3Z@eEDO+i{geU-YE@Kw8D6U z{+$qXt-kn8X9r=K5r2R4{NX2wp|Ib09SGV7+RW7YbtLVOW%zlo3@&Vm}mPs_6E zz?)X(vM))T^#-{F=@)pHiA&MR$@>_h8HXANySFYruy@8yfXdW1SQh`0G^^Pw?u(s31*0(H`-Y zdMo8gL&n9CJd6w0^KICr!?%T^NqQk7n})!_0T^DsGOh(hQfsXdq|~~!4lL41P|MB`~o2xLt)*W zc&uu&SA4Rv1v4deeCJu26;d)2><*z-6@_O#^tFQ2)g_;ROUyNxw`)cn`3M%`MpdaF zLylc!PdcNe>cVF~NAi5lt`+aNlrx4|@Sd|O;4)Y$!d8>(yS|`Ma!cUXFfBmVqO%6U z5E4=^F}V(^oSxL0!NBc)|1Kt-{5E?;1?$);&s)Q4r0ItBWSyO{tDZ@3>O0VDy`LR9_I0|ev793`B5WE%r7-Ivote|7phk%%}uKhsd>vW9` zj+Fe>Hy1v=HxZ3F`rtz0Rl0Y|a2OAD=+4R6+KYSIP#Di&gz@$r)eA02aJsTR@tA#wJD2}o4K303$v+< zONiQ3OaUvh&za`^PNBx4n$n4ex@W5)Lcgdh*$B10T=ijB4rCIx>|Jnh_xx6^bNP~+d8)^ z#;$YK)K(T{DeOjD#eN`7acJHn)OQHHmxnLI^V#FSY$&}-eMmK?WmzyEr1nc&?}7rm z2$E#G`ODaRmi(!!R=V9_IJmhGA1cJEFHWdfnlCSR(GDofvQ4dh&j=-~ z!h=XJZRuocPWzYJg{Up8mh^_{$x&=l=!rFntbTP?zC!%FU$J9Ji6*{FpTFLH3;qA{ zD;pR6gdY1PR^-q#Kg3frJegxXckMWE6Qq#q(QF@TA8QS}5Ry7=YW@~3aMd@Cs94OQ z@YCz_b)-6sNa>XoLIaiOX(`Qq+=zsch`pH16r14GJ+6|CRp1E)=^T0`^jRL#R4J04 z1-GLbx=pajlc~xg#9%yA8l-#91{G9VOH^Kn3 zW7a;LDg0N&gI|KaZTVXawp!5>&0sPNt130N-PKD+^8ZmjTjK*$cX$Zw z2vySTVTXM3KVTWJ8f~ZV%4*U~E!}{|o@p|A#kOQAM?8)5zWyoobyCniaj{tQwbG(!R(YN@lq| z9^kamT@0$q@=QqMcdwxVYRdWHc=+MffsgG)XajWxuLL<9o-OKrLO38qUw6}QIISpG z<8A-3NtGx9`oa();vwrXxWJ&^P+v97>pDrO;gR~U1LG}3>bQN--e zWkZ){fW-3LEuVi52x#gtdt_ReO*;Ir(7Lh@uoOrAVtavio`KN zZQCTaRhcDh!J9^h${IRDcIPXhR+kKvLL2S=d{}B`*}ZHP#m*!U;*${A@6~ovp=%xjgj#QP&Ups>Q-D!uxa*f(GU|PUl$v} z3}*wzVzn{TG||)eXCs1^RKnZWkjt0sPB+@#8!7|-SJIgRnkh80;p!Ymwqeuxv=|Q~ zmr%+8oiq&YiT=kyI&pWz-r=nwP!mK`HOnbHU{z~w4cHN$e}#7~15VDgtI1}U*)8S- zqaQPFx2rj0s4#dnEt@s6a@pJ_nZ!azE>f_DZ8!Z)fdH(JB2z#B;g-$72_NNQos!KO zJ;taO%8mF+c8!GiP!nNqUgy8p_*H-;m6?x$jQp6-4k7*sA;q>Er7D3X;p7cIey!%u zyBqK}GfgWKp)(#iQcJTU)6GpMzuvvcRG&5Y5EB0~!dCr~G^<;BiF_Z!`_j?<~nnp^1rGXx*Ycf);7mnaaTtXo)jOjNhmYqEbyr`u0vcK+g zavaUhq!gA5lSkuu^ZfA90Y#56rK<0Y0;ChmY#)3nx9DMXEWcPS(YiU%Zb0&E>Ie1j zsZmN0BMl820N_Fd00{n5WLevrnS1_MU{z}L*l%&8ckD61LL)bq0J)Hkqsb?Uo2--y zt((13X`L9^@?@AkQ?a$bR;@mn55y(pw`5|&gV;Wy@^QqTjYY*WbQgzfz-Vc*uUAEb zwfhcTo%F=a^?cEoWF-lft-R5Q5A&dxY916b!Qu?Kx4N-J?IGsqOt9uK!HTIo=$_l6Xxatr~n= zqeU}lNR7N&L04wG=)v0}CNnQvpL%#gxnAWWmYDzAfOx70Gsp#DcA^jO=ol1+EOe|A zOm3dw$Ku(|Z)2I`(qvWl1bno}HA`A*-*P%*YuJSRdImnr?VIOUuM3?g(BCfb29iH^ z?=u2;>clC8rSQVKq3|mx3Z*Uk`_R+Ys}v&yWjKsR((Ze3hoz9)hc%q^dR8BtNE*P3 z-G{Eb8=hG)A5w>~_Jy{GOYEiPv=uBhibqnkXT7z+{l{B!(^I5b3O~P0(#u-7ZAx}B zdd3GZi>(zl?=9Z;MJ=FoN;UoBJo$Q6V^HklO<`*XR^FGBQe;yZ9yd{wh?=VU!xRKPVro`xO4|Lo8=h?-#1haQQpx0DM@*iBd0?PkOI8hgo-ME{Wp4yu z7OsyVb{CAQr-U;tB$uaY3HLEX(MG9c-h}F{+NiI#+7zo>?0Xijtkl~bH0Fbt%tD0> z33g=e$c??cctsE~1g|f(K`LDL$3|s>29tUIu|mR!%M?Rgu9tK@ilTZ+pP?f2^IjI+ zI>b~677FL;m+EoSmbSCD2=`odE=+U~i21F5&7t-f`uA$~Vq4N(l6$A&pBxQK-~7Tu z@p_>;M3qR=7h#K&OSx{e6DYl}8~0?R7ihjI=W~*aM$8yw5mtYQI zsj9IVN^%g8m{9-k9`k!L`LCxi;9tZ4be#W&{oOqM z-yi^>Fu>rwV*5W^sedE?mM#845~Kc)LhvsU<8R>KV!}Vbo_~OjUVp>>W+(r^ia-7b zuJSkh?;`Xc_}lv?{yWvNarqnlcLnqhx*PldUl08~)885YpPBIS|HDlG` — `detail()` | Shows job metadata, progress, attempt count and latest 500 chronological log entries; unknown IDs return 404. | +| `POST /plugins/queues/jobs//cancel` — `action()` | Requires `content.write` and valid CSRF; cancels active work, audits the user action, flashes the result, and redirects to detail. | +| `POST /plugins/queues/jobs//retry` — `action()` | Same protections; retries failed/cancelled jobs. Other actions return 404. Invalid transitions display an error. | + +Signed-in users with `workspace.view` and `plugins.use` can read the queue; viewers cannot mutate it. Disabled-plugin routes return 404. Blueprint CSS/JS use the same activation/authentication gate. Templates escape titles and messages. Active panels refresh every five seconds while the tab is visible and keyboard focus is outside the panel, retaining scroll position and normal navigation. Authentication redirects stop background refresh. JavaScript is optional. + +## Internal helpers + +`bounded_text(value, label, maximum)` validates strings. `_db()` enforces activation and opens a transactional state connection. `_public(row, payload=False)` excludes worker capabilities and optionally decrypts payload for claim. `_event()` validates/records a log and bounds retention. `_owned()` verifies job state, token and lease. `_lease()` and `_counts()` validate numeric constraints. `_finish()` implements common terminal worker transitions. These helpers are implementation details; other plugins use the public methods above. + +Run integration tests from the resource checkout: + +```sh +PYTHONPATH=../web-framework ../web-framework/.venv/bin/python -m pytest tests plugins/queues/tests -q +``` diff --git a/docs/releases.md b/docs/releases.md new file mode 100644 index 0000000..a5fc8ba --- /dev/null +++ b/docs/releases.md @@ -0,0 +1,9 @@ +# Resource releases and wiki maintenance + +Keep this Git checkout beside `web-framework/`. All new plugins/themes belong here, each in a dedicated folder with JSON metadata (`id`, `name`, `description`, `version`). Independently version changed resources, bump repository `VERSION`, and document behavior in `CHANGELOG.md`. Update `docs/` function references whenever implementations change; keep README files short. + +`tools/build_catalog.py` exports `build(root)`: discovers every plugin/theme JSON manifest, builds a deterministic ZIP containing implementation, templates, assets and resource README at archive root, computes SHA-256, and writes schema-version-1 catalog entries. Unchanged packages retain their exact published archive bytes and checksums. It excludes tests, bytecode and cache folders; retains previous versioned archives. Run `python tools/build_catalog.py` from the checkout. Commit source, metadata, archives, docs, catalog and version together, then push. + +Run `python tools/sync_wikis.py` to synchronize both repositories' detailed `docs/*.md` pages to their corresponding Gitea wikis. The script accepts optional repository paths. It uses `GITEA_TOKEN` or Git's configured stored credential, enables missing wikis, initializes Home, clones each wiki into temporary storage, copies maintained pages, commits changed content, and pushes without force. It converts local Markdown page links to wiki slugs and preserves unrelated wiki pages and cleans temporary clones. Credentials are never printed or written to Git remotes. + +The framework's companion release builds and pushes a new Docker version and `latest`, then retains the current and five previous numeric framework image versions. Resource ZIP archives remain available for installed catalog versions. See the [framework release guide](https://gitea.coreplay.eu/Dymas/web-framework/wiki/releases) for the complete release procedure. diff --git a/docs/resources.md b/docs/resources.md new file mode 100644 index 0000000..f661948 --- /dev/null +++ b/docs/resources.md @@ -0,0 +1,26 @@ +# Web Framework Resources + +Complete plugin packages and theme templates for [Web Framework](https://gitea.coreplay.eu/Dymas/web-framework). + +Default catalog: `https://gitea.coreplay.eu/Dymas/web-framework-resources/raw/branch/main/catalog.json`. +Open **Config → Add resources**, search, and install a plugin or theme. Restart, then enable the plugin or select the theme. The six starter packages are also bundled in the framework for offline first startup. Queue is available only through this resource repository and requires framework 0.6.0+. + +## Folder layout + +Every resource has its own source folder: `plugins//` or `themes//`. Plugin v0.2.0 packages include `plugin.py`, `implementation.py`, `plugin.json`, README, and their page/widget templates. They depend on public framework APIs and the common shell, rather than on the shared starter implementation. Theme folders contain `theme.json` and any accompanying files; the bundled themes are presets using the framework design system. + +Every JSON manifest declares `id`, `name`, `description`, and `version`. Themes also declare `mode` (`dark`, `light`, `system`), `accent` (`violet`, `blue`, `green`), and `density` (`comfortable`, `compact`). Versioned ZIP archives contain the entire resource with files at the archive root. Catalog entries record type, metadata, relative archive path and SHA-256. + +## Persistence and updates + +Framework Compose mounts host folders `./plugins` and `./themes` outside the container, with one subfolder per resource. Image rebuilds and restarts preserve existing files. Missing defaults are copied locally from the image; no resource ZIPs are downloaded at startup. + +The app checks catalogs in the background on startup and through **Check for updates** on the configuration pages. Checks only compare metadata: they never replace code or download archives. Local versions use PEP 440 comparison; source provenance protects against same-ID packages from another publisher. Operators can disable startup checks with `[resources].check_updates_on_startup = false`. + +To update, stop the workspace, back up the resource folder, verify a newer archive against its catalog SHA-256, extract it into that folder, then restart. Keep the mounted resource folders and control-data volume. Saved plugin content and configuration remain in the control store. + +## Publish resources + +Store newly developed plugins and themes here with all required helpers, templates, and assets in their own subfolder. Increment the resource version, build its complete ZIP, and update `catalog.json` with schema version 1. Commit source, archive, and catalog together. Keep old versioned archives for existing installations. Publish catalogs and archives for unauthenticated read access; never include secrets. + +The initial six starter resources can be regenerated with `PYTHONPATH=. python tools/prepare_resources.py /path/to/web-framework-resources` from the framework repository. The exporter refreshes bundled source directories and retains independently developed resources in the catalog. `python tools/build_catalog.py` builds every source resource without replacing bundled source. Only configure trusted repositories: plugin code runs with the application user's privileges after restart. diff --git a/docs/starters.md b/docs/starters.md new file mode 100644 index 0000000..9d6e295 --- /dev/null +++ b/docs/starters.md @@ -0,0 +1,13 @@ +# Starter plugin and theme functions + +Notes, Bookmarks, and Workspace status are complete independent v0.2.0 plugin packages. Each exports `plugin` from `plugin.py` and includes namespaced templates plus implementation helpers. + +`make_collection(plugin_id, name, kind='notes')` constructs a shared collection page and summary widget. Its `index()` lists persistent items on GET. POST requires `content.write` and the framework CSRF token; it validates titles/content and writes or deletes a collection item, records an audit event, flashes the result, and redirects. Notes accept text content; bookmarks additionally require a safe HTTP(S) or root-relative link. The `intro` setting controls the page introduction. The summary widget returns the newest five items. + +`make_notes(plugin_id='notes', name='Notes')` and `make_bookmarks(plugin_id='bookmarks', name='Bookmarks')` select the corresponding collection configuration. `make_status(plugin_id='status', name='Workspace status')` builds the status page and summary widget; it reports framework version, uptime, enabled plugin count and configured connection count without resolving database passwords or contacting optional services. + +Every plugin's blueprint is registered under `/plugins//`, protected by authentication, permission, and enabled-state checks. Templates inherit the framework shell. Plugin configuration and collection data persist in the framework control store; restarting the app discovers newly installed Python packages. + +Default, Ocean, and Mint are declarative theme presets. Each `theme.json` declares ID, name, description, version, mode, accent and density. Selection uses the framework's `Themes.activate()`; effective appearance uses `Themes.appearance()`. They introduce no executable code or separate server functions. + +All resources have JSON metadata and reproducible versioned ZIP archives. See [publishing](resources.md) for package layout, checksum validation and update behavior. diff --git a/plugins/queues/README.md b/plugins/queues/README.md new file mode 100644 index 0000000..2a8117f --- /dev/null +++ b/plugins/queues/README.md @@ -0,0 +1,7 @@ +# Queue + +Persistent job queue with progress bars, per-job logs, cancellation, retries, and an internal connector for other plugins. Requires Web Framework 0.6.0+. + +Install from Config → Add resources, restart, then enable **Queue**. Producer plugins own their workers and submit/report jobs through `get_plugins().service('queues', 'queue')`. + +[Full connector and route reference](https://gitea.coreplay.eu/Dymas/web-framework-resources/wiki/queues). diff --git a/plugins/queues/connector.py b/plugins/queues/connector.py new file mode 100644 index 0000000..e1ad534 --- /dev/null +++ b/plugins/queues/connector.py @@ -0,0 +1,190 @@ +"""Durable FIFO queue. Trusted plugin workers own execution and renew leases.""" +from contextlib import contextmanager +import json +import secrets +import time +from web_framework.config import IDENTIFIER + +STATES = ('queued', 'running', 'completed', 'failed', 'cancelled') + + +def bounded_text(value, label, maximum): + if not isinstance(value, str) or not value.strip() or len(value) > maximum: + raise ValueError(f'{label} must contain 1–{maximum} characters.') + return value.strip() + + +class Queue: + """Acquire via get_plugins().service('queues', 'queue') in an app context. + + Each mutation uses the control store's SQLite transaction. Claim tokens are + private capabilities: a stale worker cannot update a reclaimed/cancelled job. + """ + def __init__(self, app): + self.app = app + self.state = app.extensions['state'] + with self.state.db() as db: + db.executescript(''' + CREATE TABLE IF NOT EXISTS queue_jobs ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + id TEXT UNIQUE NOT NULL, source TEXT NOT NULL, title TEXT NOT NULL, + payload BLOB NOT NULL, status TEXT NOT NULL DEFAULT 'queued', + completed INTEGER NOT NULL DEFAULT 0, total INTEGER NOT NULL DEFAULT 100, + created REAL NOT NULL, updated REAL NOT NULL, + token TEXT, lease_until REAL, attempts INTEGER NOT NULL DEFAULT 0); + CREATE INDEX IF NOT EXISTS queue_ready ON queue_jobs(status, source, sequence); + CREATE TABLE IF NOT EXISTS queue_logs ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + job_id TEXT NOT NULL REFERENCES queue_jobs(id) ON DELETE CASCADE, + stamp REAL NOT NULL, level TEXT NOT NULL, message TEXT NOT NULL); + CREATE INDEX IF NOT EXISTS queue_log_lookup ON queue_logs(job_id, sequence); + ''') + + @contextmanager + def _db(self): + if 'queues' not in self.app.extensions['plugins'].enabled(): + raise ValueError('The Queues plugin is disabled.') + with self.state.db() as db: + yield db + + def _public(self, row, payload=False): + item = {key: row[key] for key in ('id', 'source', 'title', 'status', 'completed', 'total', 'created', 'updated', 'attempts')} + item['progress'] = round(100 * item['completed'] / item['total'], 1) + if payload: + item['payload'] = json.loads(self.state.cipher.decrypt(row['payload'])) + return item + + def _event(self, db, job_id, message, level='info'): + message = bounded_text(message, 'Log message', 4000) + if level not in ('info', 'warning', 'error'): + raise ValueError('Choose info, warning, or error.') + db.execute('INSERT INTO queue_logs(job_id,stamp,level,message) VALUES(?,?,?,?)', (job_id, time.time(), level, message)) + db.execute('DELETE FROM queue_logs WHERE job_id=? AND sequence NOT IN (SELECT sequence FROM queue_logs WHERE job_id=? ORDER BY sequence DESC LIMIT 500)', (job_id, job_id)) + + def _owned(self, db, job_id, token): + row = db.execute('SELECT * FROM queue_jobs WHERE id=?', (job_id,)).fetchone() + if not row: + raise ValueError('Job not found.') + if row['status'] != 'running' or not isinstance(token, str) or not row['token'] or not secrets.compare_digest(row['token'], token) or row['lease_until'] <= time.time(): + raise ValueError('Job claim expired or is no longer active.') + return row + + def enqueue(self, source, title, payload=None, *, total=100): + """Submit JSON data under the producing plugin's slug; return its job ID.""" + if not isinstance(source, str) or not IDENTIFIER.fullmatch(source): + raise ValueError('Source must be a plugin slug.') + title = bounded_text(title, 'Title', 160) + self._counts(0, total) + raw = json.dumps(payload, allow_nan=False).encode() + if len(raw) > 65536: + raise ValueError('Payload exceeds 64 KiB.') + job_id, now = secrets.token_hex(16), time.time() + with self._db() as db: + db.execute('INSERT INTO queue_jobs(id,source,title,payload,total,created,updated) VALUES(?,?,?,?,?,?,?)', + (job_id, source, title, self.state.cipher.encrypt(raw), total, now, now)) + self._event(db, job_id, 'Job queued.') + return job_id + + def claim(self, source, *, lease_seconds=300): + """Atomically claim the oldest queued/expired job for a source, or None. + + Returned token must accompany worker updates. Expired jobs are delivered + again; handlers must be idempotent because execution is at least once. + """ + self._lease(lease_seconds) + now = time.time() + with self._db() as db: + row = db.execute("SELECT * FROM queue_jobs WHERE source=? AND (status='queued' OR (status='running' AND lease_until<=?)) ORDER BY sequence LIMIT 1", (source, now)).fetchone() + if row is None: + return None + token = secrets.token_hex(32) + db.execute("UPDATE queue_jobs SET status='running',token=?,lease_until=?,updated=?,attempts=attempts+1 WHERE id=?", (token, now + lease_seconds, now, row['id'])) + self._event(db, row['id'], 'Expired claim recovered.' if row['status'] == 'running' else 'Job started.') + fresh = db.execute('SELECT * FROM queue_jobs WHERE id=?', (row['id'],)).fetchone() + return {**self._public(fresh, payload=True), 'token': token} + + @staticmethod + def _lease(seconds): + if type(seconds) is not int or not 10 <= seconds <= 86400: + raise ValueError('Lease must be 10–86400 whole seconds.') + + @staticmethod + def _counts(completed, total): + if type(completed) is not int or type(total) is not int or not 1 <= total <= 2147483647 or not 0 <= completed <= total: + raise ValueError('Progress must be whole numbers with 0 <= completed <= total and total > 0.') + + def progress(self, job_id, token, completed, *, total=None, message=None, lease_seconds=300): + """Report work and renew the lease; optional message appears in job logs.""" + self._lease(lease_seconds) + with self._db() as db: + row = self._owned(db, job_id, token) + total = row['total'] if total is None else total + self._counts(completed, total) + now = time.time() + db.execute('UPDATE queue_jobs SET completed=?,total=?,updated=?,lease_until=? WHERE id=?', (completed, total, now, now + lease_seconds, job_id)) + if message is not None: + self._event(db, job_id, message) + + def log(self, job_id, token, message, *, level='info'): + """Append a bounded log entry; use progress() to renew the claim.""" + with self._db() as db: + self._owned(db, job_id, token) + self._event(db, job_id, message, level) + + def _finish(self, job_id, token, status, message): + with self._db() as db: + self._owned(db, job_id, token) + db.execute('UPDATE queue_jobs SET status=?,completed=CASE WHEN ?=\'completed\' THEN total ELSE completed END,updated=?,token=NULL,lease_until=NULL WHERE id=?', (status, status, time.time(), job_id)) + self._event(db, job_id, message, 'error' if status == 'failed' else 'info') + + def complete(self, job_id, token, message='Job completed.'): + """Mark success and set progress to 100%.""" + self._finish(job_id, token, 'completed', message) + + def fail(self, job_id, token, message='Job failed.'): + """Mark failure, preserving the last reported progress.""" + self._finish(job_id, token, 'failed', message) + + def cancel(self, job_id): + """Cancel queued/running work and invalidate its claim cooperatively.""" + with self._db() as db: + changed = db.execute("UPDATE queue_jobs SET status='cancelled',token=NULL,lease_until=NULL,updated=? WHERE id=? AND status IN ('queued','running')", (time.time(), job_id)).rowcount + if not changed: + raise ValueError('Only queued or running jobs can be cancelled.') + self._event(db, job_id, 'Job cancelled; worker claim revoked.', 'warning') + + def retry(self, job_id): + """Requeue failed/cancelled jobs with fresh progress; preserve history.""" + with self._db() as db: + changed = db.execute("UPDATE queue_jobs SET status='queued',completed=0,updated=? WHERE id=? AND status IN ('failed','cancelled')", (time.time(), job_id)).rowcount + if not changed: + raise ValueError('Only failed or cancelled jobs can be retried.') + self._event(db, job_id, 'Job queued for retry.') + + def jobs(self, *, status=None, source=None, limit=100, offset=0): + """Return paginated metadata, newest first; never expose payloads/tokens.""" + if status is not None and status not in STATES: + raise ValueError('Unknown queue status.') + if type(limit) is not int or not 1 <= limit <= 200 or type(offset) is not int or offset < 0: + raise ValueError('Invalid pagination.') + clauses, parameters = [], [] + for key, value in (('status', status), ('source', source)): + if value is not None: + clauses.append(key + '=?') + parameters.append(value) + where = ' WHERE ' + ' AND '.join(clauses) if clauses else '' + with self._db() as db: + return [self._public(row) for row in db.execute('SELECT * FROM queue_jobs' + where + ' ORDER BY sequence DESC LIMIT ? OFFSET ?', (*parameters, limit, offset))] + + def get(self, job_id): + """Read metadata; raises ValueError for a missing job.""" + with self._db() as db: + row = db.execute('SELECT * FROM queue_jobs WHERE id=?', (job_id,)).fetchone() + if row is None: + raise ValueError('Job not found.') + return self._public(row) + + def logs(self, job_id): + """Return the latest 500 entries in chronological order.""" + with self._db() as db: + return [dict(row) for row in db.execute('SELECT stamp,level,message FROM queue_logs WHERE job_id=? ORDER BY sequence', (job_id,))] diff --git a/plugins/queues/plugin.json b/plugins/queues/plugin.json new file mode 100644 index 0000000..5046ea0 --- /dev/null +++ b/plugins/queues/plugin.json @@ -0,0 +1,6 @@ +{ + "id": "queues", + "name": "Queue", + "description": "Persistent shared job queue with progress, logs, and an internal plugin connector.", + "version": "0.1.0" +} diff --git a/plugins/queues/plugin.py b/plugins/queues/plugin.py new file mode 100644 index 0000000..d0bee1f --- /dev/null +++ b/plugins/queues/plugin.py @@ -0,0 +1,68 @@ +"""Queue page and service provider; workers use the internal Python connector.""" +from flask import Blueprint, abort, current_app, flash, g, jsonify, redirect, render_template, request, url_for +from web_framework.auth import can +from web_framework.plugins import Plugin, get_plugins +from .connector import Queue, STATES + +bp = Blueprint('plugin_queues', __name__, template_folder='templates', static_folder='static') + + +def connector(): + return get_plugins().service('queues', 'queue') + + +def selection(): + status = request.args.get('status') or None + source = request.args.get('source') or None + try: + page = int(request.args.get('page', '1')) + if page < 1 or page > 1000000: + raise ValueError + jobs = connector().jobs(status=status, source=source, limit=50, offset=(page - 1) * 50) + except ValueError as error: + abort(400, description=str(error)) + return status, source, page, jobs + + +@bp.get('/') +def index(): + status, source, page, jobs = selection() + return render_template('queues/index.html', title='Queue', jobs=jobs, statuses=STATES, selected_status=status, source=source, page_number=page) + + +@bp.get('/api/jobs') +def jobs_api(): + _, _, page, jobs = selection() + return jsonify(jobs=jobs, page=page) + + +@bp.get('/jobs/') +def detail(job_id): + try: + job = connector().get(job_id) + except ValueError: + abort(404) + return render_template('queues/detail.html', title=job['title'], job=job, logs=connector().logs(job_id)) + + +@bp.post('/jobs//') +def action(job_id, action): + if not can('content.write'): + abort(403) + if action not in ('cancel', 'retry'): + abort(404) + try: + getattr(connector(), action)(job_id) + current_app.extensions['state'].audit(g.user['username'], 'queues.' + action, job_id) + flash('Job ' + ('cancelled.' if action == 'cancel' else 'queued for retry.'), 'success') + except ValueError as error: + flash(str(error), 'error') + return redirect(url_for('plugin_queues.detail', job_id=job_id)) + + +plugin = Plugin( + 'queues', 'Queue', version='0.1.0', + description='Persistent shared job queue with progress, logs, and an internal plugin connector.', + blueprint=bp, endpoint='plugin_queues.index', + services={'queue': lambda: Queue(current_app._get_current_object())}, +) diff --git a/plugins/queues/static/queues.css b/plugins/queues/static/queues.css new file mode 100644 index 0000000..c56598b --- /dev/null +++ b/plugins/queues/static/queues.css @@ -0,0 +1,23 @@ +.queue-panel { margin-bottom: 24px; } +.queue-filters { display: flex; align-items: end; flex-wrap: wrap; gap: 12px; } +.queue-filters label { display: grid; gap: 8px; flex: 1 1 180px; font-size: 12px; } +.queue-refresh { margin: 16px 0; font-size: 12px; } +.queue-panel progress { width: 100%; min-width: 100px; height: 12px; accent-color: var(--accent); } +.queue-panel small { display: block; margin-top: 6px; color: var(--muted); } +.queue-progress { margin: 18px 0 10px; } +.queue-actions { margin: 20px 0; } +.queue-pagination { display: flex; align-items: center; gap: 16px; margin-top: 20px; } +.queue-logs { list-style: none; padding: 0; margin: 18px 0; max-height: 50vh; overflow: auto; } +.queue-logs li { display: flex; flex-wrap: wrap; gap: 12px; padding: 12px 0; border-bottom: 1px solid var(--line); white-space: pre-wrap; overflow-wrap: anywhere; } +.queue-logs time { color: var(--muted); font-size: 12px; } + +.queue-panel th, .queue-panel td { white-space: nowrap; } +.queue-panel td:first-child { white-space: normal; min-width: 160px; } +@media (max-width: 640px) { + .queue-panel thead { display: none; } + .queue-panel tbody { display: grid; gap: 14px; } + .queue-panel tr { display: grid; gap: 12px; padding: 16px 0; border-top: 1px solid var(--line); } + .queue-panel td { display: grid; grid-template-columns: 65px minmax(0, 1fr); gap: 12px; padding: 0; border: 0; white-space: normal; font-size: 12px; } + .queue-panel td::before { content: attr(data-label); color: var(--muted); } + .queue-panel .badge { width: fit-content; } +} diff --git a/plugins/queues/static/queues.js b/plugins/queues/static/queues.js new file mode 100644 index 0000000..0b4947c --- /dev/null +++ b/plugins/queues/static/queues.js @@ -0,0 +1,15 @@ +// Replace the queue panel only; retain keyboard focus and scroll position. +const refreshQueue = async () => { + const panel = document.querySelector('[data-queue-refresh]'); + if (!panel || document.hidden || panel.contains(document.activeElement)) return; + try { + const response = await fetch(window.location.href, {redirect: 'error'}); + if (!response.ok) return; + const documentNext = new DOMParser().parseFromString(await response.text(), 'text/html'); + const nextPanel = documentNext.querySelector('.queue-panel'); + if (nextPanel) panel.replaceWith(nextPanel); + } catch (_) { + // Manual refresh remains available when a connection or session expires. + } +}; +setInterval(refreshQueue, 5000); diff --git a/plugins/queues/templates/queues/detail.html b/plugins/queues/templates/queues/detail.html new file mode 100644 index 0000000..eae3f7e --- /dev/null +++ b/plugins/queues/templates/queues/detail.html @@ -0,0 +1,26 @@ +{% extends 'base.html' %} +{% block head %} + + +{% endblock %} +{% block content %} +

QUEUE JOB

{{ job.title }}

{{ job.source }}

Back to Queue
+
+

{{ job.status }} · Attempt {{ job.attempts }}

+

Progress

+ {{ job.progress }}% +

{{ job.progress }}% · {{ job.completed }} of {{ job.total }} completed

+

Created {{ job.created|timestamp }} · Updated {{ job.updated|timestamp }}

+ {% if can('content.write') and job.status != 'completed' %} +
+ + +
+ {% endif %} +

Job log

+
    + {% for log in logs %}
  1. {{ log.level }} {{ log.message }}
  2. {% else %}
  3. No log entries yet.
  4. {% endfor %} +
+

Showing the latest 500 entries. Refresh now

+
+{% endblock %} diff --git a/plugins/queues/templates/queues/index.html b/plugins/queues/templates/queues/index.html new file mode 100644 index 0000000..a3f82ba --- /dev/null +++ b/plugins/queues/templates/queues/index.html @@ -0,0 +1,32 @@ +{% extends 'base.html' %} +{% block head %} + + +{% endblock %} +{% block content %} +

PLUGIN JOBS

Queue

Follow queued work, progress, and job logs across your plugins.

+
+
+ + + + Reset +
+

Updates every 5 seconds. Refresh now

+ {% if jobs %} +
+ {% for job in jobs %} + + + + + {% endfor %} +
JobPluginStatusProgressUpdated
{{ job.title }}{{ job.source }}{{ job.status }}
{{ job.progress }}%{{ job.progress }}% · {{ job.completed }}/{{ job.total }}
{{ job.updated|timestamp }}
+ {% else %}

No jobs here yet

Jobs submitted by connected plugins will appear here. Try clearing your filters.

{% endif %} + +
+{% endblock %} diff --git a/plugins/queues/tests/browser_check.py b/plugins/queues/tests/browser_check.py new file mode 100644 index 0000000..d1cc085 --- /dev/null +++ b/plugins/queues/tests/browser_check.py @@ -0,0 +1,86 @@ +"""Exercise Queue in Chromium, including polling, subpath, and mobile layout.""" +import logging +import os +from pathlib import Path +import tempfile +import threading +from playwright.sync_api import expect, sync_playwright +from werkzeug.serving import make_server +from web_framework import create_app + + +def main(): + logging.getLogger('werkzeug').setLevel(logging.ERROR) + with tempfile.TemporaryDirectory(prefix='queues-browser-') as temporary: + root = Path(temporary) + server = make_server('127.0.0.1', 0, lambda environ, start_response: []) + base = f'http://127.0.0.1:{server.server_port}/projectx' + app = create_app(overrides={ + 'app': {'base_url': base}, + 'runtime': {'data_dir': str(root / 'state'), 'plugin_dir': str(Path(__file__).resolve().parents[2]), 'theme_dir': str(root / 'themes')}, + 'plugins': {'enabled': ['queues']}, + 'resources': {'check_updates_on_startup': False}, + 'logging': {'console': False}, + }) + server.app = app + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + try: + queue = app.extensions['plugins'].service('queues', 'queue') + job_id = queue.enqueue('reports', 'Monthly report', total=10) + with sync_playwright() as playwright: + options = {'headless': True} + if os.environ.get('PLAYWRIGHT_CHROMIUM_EXECUTABLE'): + options['executable_path'] = os.environ['PLAYWRIGHT_CHROMIUM_EXECUTABLE'] + browser = playwright.chromium.launch(**options) + page = browser.new_page() + errors = [] + page.on('pageerror', lambda error: errors.append(str(error))) + page.on('console', lambda message: errors.append(message.text) if message.type == 'error' else None) + page.goto(base + '/plugins/queues/') + page.get_by_label('Username', exact=True).fill('admin') + password = (root / 'state/initial-admin-password').read_text().strip() + page.get_by_label('Password', exact=True).fill(password) + page.get_by_role('button', name='Sign in', exact=True).click() + page.get_by_label('Current password', exact=True).fill(password) + page.get_by_label('New password', exact=True).fill('Queue-browser-password!42') + page.get_by_label('Confirm new password', exact=True).fill('Queue-browser-password!42') + page.get_by_role('button', name='Change password', exact=True).click() + page.goto(base + '/plugins/queues/') + expect(page.get_by_role('link', name='Monthly report')).to_be_visible() + page.get_by_role('link', name='Monthly report').click() + expect(page.get_by_text('Job queued.', exact=True)).to_be_visible() + job = queue.claim('reports') + queue.progress(job_id, job['token'], 5, message='Halfway through') + expect(page.get_by_text('Halfway through', exact=True)).to_be_visible(timeout=10000) + expect(page.get_by_role('progressbar')).to_have_attribute('value', '5') + queue.complete(job_id, job['token']) + expect(page.get_by_text('Job completed.', exact=True)).to_be_visible(timeout=10000) + expect(page.locator('.queue-panel .badge').first).to_have_text('completed') + assert not page.locator('[data-queue-refresh]').count() + failed = queue.enqueue('reports', 'Retry this report') + claimed = queue.claim('reports') + queue.fail(failed, claimed['token'], 'Example failure') + page.goto(base + '/plugins/queues/jobs/' + failed) + page.get_by_role('button', name='Retry job', exact=True).click() + expect(page.get_by_role('button', name='Cancel job')).to_be_visible() + page.get_by_role('button', name='Cancel job', exact=True).click() + expect(page.get_by_role('button', name='Retry job')).to_be_visible() + assert queue.get(failed)['status'] == 'cancelled' + page.set_viewport_size({'width': 390, 'height': 844}) + page.goto(base + '/plugins/queues/?status=completed&source=reports') + expect(page.get_by_role('link', name='Monthly report')).to_be_visible() + expect(page.get_by_role('link', name='Retry this report')).to_have_count(0) + assert page.evaluate('document.documentElement.scrollWidth <= window.innerWidth') + page.screenshot(path='/tmp/queue-mobile-preview.png', full_page=True) + assert not errors, errors + browser.close() + finally: + server.shutdown() + worker.join() + server.server_close() + print('Queue browser checks passed: polling, completion, logs, retry/cancel, filters, mobile, subpath, CSP.') + + +if __name__ == '__main__': + main() diff --git a/plugins/queues/tests/test_queue.py b/plugins/queues/tests/test_queue.py new file mode 100644 index 0000000..fc3ee81 --- /dev/null +++ b/plugins/queues/tests/test_queue.py @@ -0,0 +1,146 @@ +"""Connector, durable worker claims, and browser access integration.""" +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +import pytest +from web_framework import create_app + + +@pytest.fixture +def app(tmp_path, monkeypatch): + monkeypatch.setenv('WEB_FRAMEWORK_DATA_DIR', str(tmp_path / 'state')) + app = create_app(overrides={ + 'runtime': {'plugin_dir': str(Path(__file__).resolve().parents[2]), 'theme_dir': str(tmp_path / 'themes')}, + 'plugins': {'enabled': ['queues']}, + 'resources': {'check_updates_on_startup': False}, + }) + app.testing = True + return app + + +@pytest.fixture +def queue(app): + return app.extensions['plugins'].service('queues', 'queue') + + +def authenticate(app): + client = app.test_client() + password = (app.extensions['state'].directory / 'initial-admin-password').read_text().strip() + client.get('/login') + with client.session_transaction() as session: + token = session['csrf'] + client.post('/login', data={'username': 'admin', 'password': password, 'csrf_token': token}) + client.post('/account', data={'current_password': password, 'new_password': 'A-secure-password!42', 'confirm_password': 'A-secure-password!42', 'csrf_token': csrf(client)}) + return client + + +def csrf(client): + with client.session_transaction() as session: + return session['csrf'] + + +def test_lifecycle_payload_progress_logs_and_restart(app, queue): + job_id = queue.enqueue('reports', '', {'rows': [1, 2]}, total=2) + claimed = queue.claim('reports') + assert claimed['id'] == job_id and claimed['payload'] == {'rows': [1, 2]} + assert claimed['attempts'] == 1 + assert queue.claim('reports') is None + queue.progress(job_id, claimed['token'], 1, message='') + assert queue.get(job_id)['progress'] == 50 + queue.complete(job_id, claimed['token']) + assert queue.get(job_id)['status'] == 'completed' + assert queue.get(job_id)['progress'] == 100 + assert len(queue.logs(job_id)) == 4 + restarted = create_app(overrides=app.extensions['workspace']['config']) + restored = restarted.extensions['plugins'].service('queues', 'queue') + assert restored.get(job_id)['status'] == 'completed' + assert restored.logs(job_id) == queue.logs(job_id) + with pytest.raises(ValueError): + queue.progress(job_id, claimed['token'], 0) + with pytest.raises(ValueError): + queue.retry(job_id) + client = authenticate(app) + response = client.get('/plugins/queues/jobs/' + job_id) + assert b'<script>' in response.data and b'