From ca169be0fb764a58ec9f83c09061bf4ee421db08 Mon Sep 17 00:00:00 2001 From: Disco DeDisco Date: Thu, 21 May 2026 14:24:41 -0400 Subject: [PATCH] =?UTF-8?q?fix:=20CI=20Postgres=20teardown=20=E2=80=94=20`?= =?UTF-8?q?RobustCompressorTestRunner.teardown=5Fdatabases`=20now=20force-?= =?UTF-8?q?closes=20lingering=20connections=20before=20Django's=20DROP.=20?= =?UTF-8?q?CI=20step=20`test-UTs-n-ITs`=20(`python=20manage.py=20test=20ap?= =?UTF-8?q?ps`,=20full=20suite=20incl.=20channels-tagged=20tests)=20was=20?= =?UTF-8?q?failing=20post-test=20even=20when=20all=201165=20tests=20passed?= =?UTF-8?q?=20=E2=80=94=20`psycopg2.errors.ObjectInUse:=20database=20"test?= =?UTF-8?q?=5Fpython=5Ftdd=5Ftest"=20is=20being=20accessed=20by=20other=20?= =?UTF-8?q?users=20/=20DETAIL:=20There=20is=201=20other=20session=20using?= =?UTF-8?q?=20the=20database`.=20Two-step=20leak:=20(1)=20`core.settings.D?= =?UTF-8?q?ATABASES['default']['conn=5Fmax=5Fage']=3D600`=20keeps=20Postgr?= =?UTF-8?q?es=20connections=20alive=20in=20the=20per-thread=20pool=20for?= =?UTF-8?q?=2010=20min=20(prod-perf=20default);=20(2)=20Channels'=20`datab?= =?UTF-8?q?ase=5Fsync=5Fto=5Fasync`=20(16=20call=20sites=20across=20`apps.?= =?UTF-8?q?epic.tests.integrated.test=5Fconsumers`'s=20`CursorMoveConsumer?= =?UTF-8?q?Test`=20+=20`SigHoverConsumerTest`)=20runs=20in=20a=20process-w?= =?UTF-8?q?ide=20asgiref=20threadpool=20=E2=80=94=20each=20worker=20thread?= =?UTF-8?q?=20accumulates=20its=20own=20DB=20connection=20that=20outlives?= =?UTF-8?q?=20the=20test=20+=20sits=20idle=20in=20the=20pool=20when=20tear?= =?UTF-8?q?down=20fires.=20Postgres=20refuses=20DROP=20while=20ANY=20sessi?= =?UTF-8?q?on=20targets=20the=20row.=20Local=20dev=20unaffected:=20`--excl?= =?UTF-8?q?ude-tag=3Dchannels`=20skips=20the=20consumer=20tests=20+=20SQLi?= =?UTF-8?q?te=20has=20no=20DROP=20step.=20**Fix**=20lives=20entirely=20in?= =?UTF-8?q?=20`core/runner.py`'s=20teardown=20override=20=E2=80=94=20`conn?= =?UTF-8?q?ections.close=5Fall()`=20covers=20the=20main=20thread's=20runne?= =?UTF-8?q?r=20connection;=20iterating=20`old=5Fconfig`=20+=20running=20`S?= =?UTF-8?q?ELECT=20pg=5Fterminate=5Fbackend(pid)=20FROM=20pg=5Fstat=5Facti?= =?UTF-8?q?vity=20WHERE=20datname=20=3D=20%s=20AND=20pid=20<>=20pg=5Fbacke?= =?UTF-8?q?nd=5Fpid()`=20against=20each=20test=20DB=20kicks=20any=20worker?= =?UTF-8?q?-thread=20session=20still=20pinning=20the=20row.=20Both=20safe?= =?UTF-8?q?=20on=20a=20green=20run=20(DB=20about=20to=20be=20dropped=20any?= =?UTF-8?q?way)=20+=20scoped=20to=20`vendor=20=3D=3D=20"postgresql"`=20so?= =?UTF-8?q?=20SQLite=20local-dev=20is=20a=20clean=20no-op.=20Prod=20`CONN?= =?UTF-8?q?=5FMAX=5FAGE=3D600`=20untouched=20=E2=80=94=20fix=20lives=20in?= =?UTF-8?q?=20the=20test=20runner,=20NOT=20in=20settings.=2019/19=20lyric?= =?UTF-8?q?=20UTs=20green=20via=20the=20new=20runner=20path=20(smoke=20ver?= =?UTF-8?q?ify=20the=20override=20is=20benign=20on=20SQLite);=20Postgres-s?= =?UTF-8?q?ide=20validated=20next=20CI=20run.=20Trap=20captured:=20[[feedb?= =?UTF-8?q?ack-test-teardown-conn-leak]]=20=E2=80=94=20symptom=20signature?= =?UTF-8?q?=20`Ran=20NNNN=20tests=20/=20OK=20/=20Destroying.../ObjectInUse?= =?UTF-8?q?:=20...`=20belongs=20in=20CI-fail=20triage=20notes=20so=20futur?= =?UTF-8?q?e=20flakes=20get=20diagnosed=20in=20seconds=20instead=20of=20te?= =?UTF-8?q?st-hunting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 (1M context) --- src/core/runner.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/src/core/runner.py b/src/core/runner.py index b800ea7..2dc9327 100644 --- a/src/core/runner.py +++ b/src/core/runner.py @@ -58,3 +58,42 @@ class RobustCompressorTestRunner(DiscoverRunner): except PermissionError: time.sleep(0.05) raise CompressorFileStorage.save = _robust_save + + def teardown_databases(self, old_config, **kwargs): + """Force-close any lingering DB connections before Django issues + `DROP DATABASE`. Without this, CI fails post-test w. Postgres' + `ObjectInUse: database "..." is being accessed by other users` + even on a fully-green run. + + Root cause: `core.settings.DATABASES['default']['conn_max_age'] + = 600` keeps Postgres connections alive in the per-thread pool + for 10 min. Channels' `database_sync_to_async` (used in + `apps.epic.tests.integrated.test_consumers`'s `CursorMoveConsumerTest` + + `SigHoverConsumerTest`) runs in a global asgiref threadpool; + each worker thread accumulates its own DB connection that + outlives the test + sits idle in the pool when teardown fires. + Postgres refuses the drop while ANY session targets the row. + + Two-step fix — `connections.close_all()` covers the main + thread (test runner's own connection); `pg_terminate_backend` + targets any worker-thread session still pinning the test DB. + Both are safe on a green test run: the DB is about to be + dropped anyway, and prod's `CONN_MAX_AGE=600` stays untouched + (the fix lives in the test runner, not in `settings.py`). + + Local dev (SQLite, no DROP step) is unaffected; CI Postgres + is where the leak was visible. See [[feedback-test-teardown-conn-leak]]. + """ + from django.db import connections + connections.close_all() + for connection, _, _ in old_config: + if connection.vendor == "postgresql": + test_db_name = connection.settings_dict["NAME"] + with connection._nodb_cursor() as cursor: + cursor.execute( + "SELECT pg_terminate_backend(pid) " + "FROM pg_stat_activity " + "WHERE datname = %s AND pid <> pg_backend_pid()", + [test_db_name], + ) + super().teardown_databases(old_config, **kwargs)