From f9cd08a510739336e2ae27d6a7548b1befab0348 Mon Sep 17 00:00:00 2001 From: Disco DeDisco Date: Wed, 13 May 2026 14:12:38 -0400 Subject: [PATCH] =?UTF-8?q?CI:=20FT=20stages=20run=20in=20parallel,=20--pa?= =?UTF-8?q?rallel=20dropped=20intra-stage=20=E2=80=94=20bud-btn=20click=20?= =?UTF-8?q?sites=20bypass=20scroll-into-view=20;=20.woodpecker/main.yaml?= =?UTF-8?q?=20restructures=20the=20FT=20split=20=E2=80=94=20test-FTs-non-r?= =?UTF-8?q?oom=20+=20test-FTs-room=20now=20both=20`depends=5Fon:=20test-tw?= =?UTF-8?q?o-browser-FTs`=20(instead=20of=20room=20serially=20depending=20?= =?UTF-8?q?on=20non-room)=20so=20they=20fan=20out=20+=20run=20concurrently?= =?UTF-8?q?,=20each=20w.=20its=20own=20`DATABASE=5FURL:=20sqlite:////tmp/t?= =?UTF-8?q?est=5Fdb=5F{non=5Froom,room}.sqlite3`=20outside=20the=20shared?= =?UTF-8?q?=20workspace=20mount=20so=20the=20two=20stages=20can't=20see=20?= =?UTF-8?q?each=20other's=20SQLite=20file=20+=20the=20#296=20EOFError-on-h?= =?UTF-8?q?alf-created-test-db=20blocker=20is=20gone;=20`--parallel`=20fla?= =?UTF-8?q?g=20dropped=20from=20both=20`manage.py=20test`=20invocations=20?= =?UTF-8?q?because=20the=20empirical=20wall-clock=20from=20#302-304=20(151?= =?UTF-8?q?=20tests=20in=20~42=20min,=2083=20tests=20in=20~16=20min=20?= =?UTF-8?q?=E2=80=94=20~16-17s/test=20avg=20either=20way)=20shows=20~1-1.5?= =?UTF-8?q?x=20speedup=20at=20best=20=E2=80=94=20Firefox=20spawn=20cost=20?= =?UTF-8?q?(~3-5s=20cold-start=20=C3=97=2038=20tests/worker)=20+=20RAM=20p?= =?UTF-8?q?ressure=20(4=20headless=20Firefoxes=20=E2=89=88=201.5-2GB=20wor?= =?UTF-8?q?king=20set=20on=20a=20quadcore=20DO=20droplet)=20+=20SQLite=20f?= =?UTF-8?q?ile-lock=20contention=20eat=20most=20of=20the=20gain=20the=20co?= =?UTF-8?q?res=20would=20otherwise=20give,=20while=20the=20contention=20am?= =?UTF-8?q?plifies=20every=20transient-DOM=20flake=20we've=20spent=20the?= =?UTF-8?q?=20last=202=20days=20chasing=20(login-race=20in=20#300/303=20?= =?UTF-8?q?=E2=86=92=20054b0aa=20+=20ad0041d,=20gecko-perms=20in=20#302/30?= =?UTF-8?q?3=20=E2=86=92=20ad0041d,=20ElementNotInteractable=20in=20#304?= =?UTF-8?q?=20=E2=86=92=20this=20commit,=20Jasmine-timeout=20in=20#303=20?= =?UTF-8?q?=E2=86=92=20ad0041d);=20stage-level=20parallelism=20gets=20back?= =?UTF-8?q?=20the=20wall-clock=20reduction=20(~16min=20savings=20vs=20seri?= =?UTF-8?q?al=20stages)=20w.o.=20amplifying=20within-stage=20contention=20?= =?UTF-8?q?=E2=80=94=20net=20wall=20clock=20should=20be=20~60-65min=20for?= =?UTF-8?q?=20both=20FT=20stages=20running=20concurrently=20vs=20~58min=20?= =?UTF-8?q?today,=20but=20flake=20exposure=20drops=20dramatically;=20downs?= =?UTF-8?q?tream=20`screendumps`=20+=20`build-and-push`=20already=20list?= =?UTF-8?q?=20both=20FT=20steps=20in=20their=20`depends=5Fon`=20so=20they?= =?UTF-8?q?=20naturally=20gate=20on=20the=20slower=20of=20the=20two=20;=20?= =?UTF-8?q?test=5Fcore=5Fbud=5Fbtn.py=20wraps=204=20unwrapped=20`find=5Fel?= =?UTF-8?q?ement(...).click()`=20sites=20in=20`wait=5Ffor(execute=5Fscript?= =?UTF-8?q?("arguments[0].click()",=20btn))`=20=E2=80=94=20the=20`=5Fopen?= =?UTF-8?q?=5Fpanel=5Fand=5Finvite`=20helper=20feeding=206=20GatekeeperBud?= =?UTF-8?q?BtnAsyncInviteTest=20tests=20+=20the=20standalone=20GatekeeperB?= =?UTF-8?q?udBtnDuplicateInviteErrorTest=20test=5Fduplicate=5Finvite=5Fsho?= =?UTF-8?q?ws=5Ferror=5Fbrief=5Fand=5Ffyi=5Fflashes=5Fslot=20click=20+=20b?= =?UTF-8?q?oth=20`.note-banner--duplicate=20.note-banner=5F=5Ffyi`=20click?= =?UTF-8?q?s=20(one=20on=20the=20share-flow=20at=20line=20433,=20one=20on?= =?UTF-8?q?=20the=20gatekeeper-invite-flow=20at=20line=20622)=20=E2=80=94?= =?UTF-8?q?=20pipeline=20#304=20errored=20on=20the=20OK=20button=20click?= =?UTF-8?q?=20w.=20`ElementNotInteractableException:=20Element=20=20could=20not=20be=20scrolled=20into?= =?UTF-8?q?=20view`=20because=20the=20post-send=5Fkeys=20autocomplete=20dr?= =?UTF-8?q?opdown=20on=20`=5Fbud=5Finvite=5Fpanel.html`=20briefly=20overla?= =?UTF-8?q?pped=20the=20OK=20button=20under=20CI=20contention=20so=20Firef?= =?UTF-8?q?ox=20refused=20the=20scroll-into-view;=20same=20pattern=20+=20s?= =?UTF-8?q?ame=20fix=20shape=20as=20confirm=5Fguard=20in=20base.py=20?= =?UTF-8?q?=E2=80=94=20execute=5Fscript=20bypasses=20Selenium's=20scroll-i?= =?UTF-8?q?nto-view=20gate=20entirely=20+=20wait=5Ffor=20absorbs=20any=20l?= =?UTF-8?q?eftover=20transient=20state=20via=20WebDriverException=20retry?= =?UTF-8?q?=20;=20commit=20only=20ships=20the=20test-side=20fix=20+=20CI?= =?UTF-8?q?=20restructure;=20if=20the=20CI=20changes=20work=20as=20intende?= =?UTF-8?q?d=20(green=20pipeline=20#305)=20the=20docker-rebuild=20option?= =?UTF-8?q?=20for=20python-tdd-ci:latest=20still=20stands=20as=20the=20nex?= =?UTF-8?q?t=20~5min/pipeline=20win,=20separately=20filed=20in=20project?= =?UTF-8?q?=5Fci=5Fremove=5Fpip=5Finstall=5Fdeferred.md=20=E2=80=94=20TDD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Code architected by Disco DeDisco Git commit message Co-Authored-By: Claude Opus 4.7 --- .woodpecker/main.yaml | 50 ++++++++++++----------- src/functional_tests/test_core_bud_btn.py | 40 +++++++++++++----- 2 files changed, 57 insertions(+), 33 deletions(-) diff --git a/.woodpecker/main.yaml b/.woodpecker/main.yaml index 780f59a..f66bab3 100644 --- a/.woodpecker/main.yaml +++ b/.woodpecker/main.yaml @@ -64,31 +64,29 @@ steps: - "requirements.txt" - ".woodpecker/main.yaml" - # ── FT split (sequential for now) ───────────────────────────────────── + # ── FT split (stage-parallel, intra-stage sequential) ───────────────── # # test_game_room_* is the heaviest cluster — 9 Selenium-driven room-flow # FTs that historically dominate the FT step wall-clock (~70% of the # ~40-min single-step runs). Split off into its own step (`test-FTs-room`) # so the partition is visible in the pipeline view; the non-room bucket - # runs first as `test-FTs-non-room`. test-FTs-non-room depends on - # test-two-browser-FTs (collectstatic'd assets in shared workspace); - # test-FTs-room depends on test-FTs-non-room. + # is `test-FTs-non-room`. Both depend on test-two-browser-FTs only, so + # they fan out + run concurrently. # - # NOTE — currently sequential, not parallel. Both FT steps share the - # workspace AND fall back to SQLite (only test-UTs-n-ITs has - # `DATABASE_URL` pointing at the postgres service). When they ran - # concurrently (both depending on test-two-browser-FTs), the second - # step to start hit a half-created `src/test_db.sqlite3` and prompted - # "Type 'yes' to delete the existing test database" → EOFError under - # non-interactive CI stdin. See pipeline run #296. + # The previous SQLite-collision blocker (pipeline #296: second step + # started against the first step's half-created `src/test_db.sqlite3` + # → Django interactive prompt → EOFError under non-interactive CI + # stdin) is resolved by giving each step a distinct `DATABASE_URL` + # pointing at its own sqlite file under /tmp — outside the shared + # workspace mount so the two stages can't see each other's DB. # - # To re-parallelise later, give each step its own DB: - # - point `DATABASE_URL` at distinct sqlite paths per step - # (e.g. `sqlite:////tmp/test_db_room.sqlite3` vs `_non_room`), or - # - have each step point at its own postgres DB (the existing - # `postgres` service already exposes one; spin up a second or - # `CREATE DATABASE` per step). - # Until then, test-FTs-room runs after test-FTs-non-room. + # `--parallel` is dropped from both steps. Empirically (pipelines + # #302-304) it was giving ~1-1.5x speedup at most on these Selenium + # FTs — Firefox spawn cost + RAM pressure + SQLite file-lock contention + # eat most of the gain — while amplifying every transient-DOM flake + # (login-race, gecko-perms, ElementNotInteractable, Jasmine-timeout). + # Stage-level parallelism gives the same wall-clock reduction without + # contention amplification. - name: test-FTs-non-room image: gitea.earthmanrpg.me/discoman/python-tdd-ci:latest @@ -96,6 +94,9 @@ steps: - test-two-browser-FTs environment: HEADLESS: 1 + # /tmp path (not workspace-relative) so the parallel test-FTs-room + # step can't see this DB + vice versa. See split-rationale above. + DATABASE_URL: sqlite:////tmp/test_db_non_room.sqlite3 CELERY_BROKER_URL: redis://redis:6379/0 REDIS_URL: redis://redis:6379/1 STRIPE_SECRET_KEY: @@ -109,7 +110,7 @@ steps: # Every FT file EXCEPT test_game_room_* — that cluster runs in # test-FTs-room. Channels + two-browser tags already covered upstream. # `ls | grep -v | sed` enumerates module dotted-paths from filenames. - - python manage.py test --parallel --exclude-tag=channels --exclude-tag=two-browser $(ls functional_tests/test_*.py | grep -v 'test_game_room_' | sed 's|/|.|g;s|\.py||') + - python manage.py test --exclude-tag=channels --exclude-tag=two-browser $(ls functional_tests/test_*.py | grep -v 'test_game_room_' | sed 's|/|.|g;s|\.py||') when: - event: push path: @@ -120,9 +121,12 @@ steps: - name: test-FTs-room image: gitea.earthmanrpg.me/discoman/python-tdd-ci:latest depends_on: - - test-FTs-non-room + - test-two-browser-FTs environment: HEADLESS: 1 + # /tmp path (not workspace-relative) so test-FTs-non-room can't see + # this DB + vice versa. See split-rationale above. + DATABASE_URL: sqlite:////tmp/test_db_room.sqlite3 CELERY_BROKER_URL: redis://redis:6379/0 REDIS_URL: redis://redis:6379/1 STRIPE_SECRET_KEY: @@ -135,9 +139,9 @@ steps: - cd ./src # Heavy Selenium room flows — 9 files (deck_contrib, gatekeeper, # invite, select_role/sea/sig/sky, tray, tray_tooltip) isolated - # into their own sub-step. Runs after test-FTs-non-room (shared - # SQLite test DB; see split-rationale comment above). - - python manage.py test --parallel --exclude-tag=channels --exclude-tag=two-browser $(ls functional_tests/test_game_room_*.py | sed 's|/|.|g;s|\.py||') + # into their own sub-step. Runs in parallel w. test-FTs-non-room + # (distinct DATABASE_URL paths under /tmp; see split-rationale). + - python manage.py test --exclude-tag=channels --exclude-tag=two-browser $(ls functional_tests/test_game_room_*.py | sed 's|/|.|g;s|\.py||') when: - event: push path: diff --git a/src/functional_tests/test_core_bud_btn.py b/src/functional_tests/test_core_bud_btn.py index b673dbe..e0b66ec 100644 --- a/src/functional_tests/test_core_bud_btn.py +++ b/src/functional_tests/test_core_bud_btn.py @@ -427,9 +427,12 @@ class BudBtnDuplicateShareErrorTest(FunctionalTest): self.assertNotIn("bud-duplicate-flash", chip.get_attribute("class") or "") # FYI dismisses + applies flash class - self.browser.find_element( - By.CSS_SELECTOR, ".note-banner--duplicate .note-banner__fyi" - ).click() + self.wait_for(lambda: self.browser.execute_script( + "arguments[0].click()", + self.browser.find_element( + By.CSS_SELECTOR, ".note-banner--duplicate .note-banner__fyi" + ), + )) self.wait_for(lambda: self.assertEqual( self.browser.find_elements(By.CSS_SELECTOR, ".note-banner--duplicate"), [], @@ -503,9 +506,17 @@ class GatekeeperBudBtnAsyncInviteTest(FunctionalTest): lambda: self.browser.find_element(By.ID, "id_recipient") ) recipient_input.send_keys(recipient) - self.browser.find_element( - By.CSS_SELECTOR, "#id_bud_panel .btn.btn-confirm" - ).click() + # wait_for + execute_script bypasses Selenium's scroll-into-view + # gate — under CI contention the post-send_keys autocomplete dropdown + # can briefly overlap the OK button + Firefox refuses the click w. + # ElementNotInteractableException (pipeline #304). Same shape as + # confirm_guard in base.py. + self.wait_for(lambda: self.browser.execute_script( + "arguments[0].click()", + self.browser.find_element( + By.CSS_SELECTOR, "#id_bud_panel .btn.btn-confirm" + ), + )) return bud_btn def test_invite_creates_room_invite(self): @@ -579,7 +590,13 @@ class GatekeeperBudBtnDuplicateInviteErrorTest(FunctionalTest): btn.click() recipient = self.wait_for(lambda: self.browser.find_element(By.ID, "id_recipient")) recipient.send_keys("alice@test.io") - self.browser.find_element(By.CSS_SELECTOR, "#id_bud_panel .btn.btn-confirm").click() + # See _open_panel_and_invite (above) for the bypass rationale. + self.wait_for(lambda: self.browser.execute_script( + "arguments[0].click()", + self.browser.find_element( + By.CSS_SELECTOR, "#id_bud_panel .btn.btn-confirm" + ), + )) title = self.wait_for(lambda: self.browser.find_element( By.CSS_SELECTOR, ".note-banner--duplicate .note-banner__title" @@ -600,9 +617,12 @@ class GatekeeperBudBtnDuplicateInviteErrorTest(FunctionalTest): ) self.assertNotIn("bud-duplicate-flash", slot.get_attribute("class") or "") - self.browser.find_element( - By.CSS_SELECTOR, ".note-banner--duplicate .note-banner__fyi" - ).click() + self.wait_for(lambda: self.browser.execute_script( + "arguments[0].click()", + self.browser.find_element( + By.CSS_SELECTOR, ".note-banner--duplicate .note-banner__fyi" + ), + )) self.wait_for(lambda: self.assertEqual( self.browser.find_elements(By.CSS_SELECTOR, ".note-banner--duplicate"), [],