From 14b20cecd6b96c2dc0071501ca183f0f04fa34cf Mon Sep 17 00:00:00 2001 From: Mohith26 <109003762+Mohith26@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:59:14 -0500 Subject: [PATCH] Add option to detach old partitions instead of dropping them Adds PostgresSchemaEditor.detach_partition() and PostgresPartition.detach(), lets PostgresPartitioningPlan.apply() detach partitions (optionally with DETACH PARTITION ... CONCURRENTLY on PostgreSQL 14+) and exposes it as `pgpartition --detach {no,sequentially,concurrently}`. The detached tables and their rows are kept. --- docs/source/table_partitioning.rst | 21 +++++ psqlextra/backend/schema.py | 54 +++++++++++ psqlextra/management/commands/pgpartition.py | 23 ++++- psqlextra/partitioning/partition.py | 17 ++++ psqlextra/partitioning/plan.py | 78 +++++++++++++--- tests/test_management_command_partition.py | 42 +++++++++ tests/test_partitioning_time.py | 96 ++++++++++++++++++++ tests/test_schema_editor_partitioning.py | 95 ++++++++++++++++++- 8 files changed, 409 insertions(+), 17 deletions(-) diff --git a/docs/source/table_partitioning.rst b/docs/source/table_partitioning.rst index 1c36db0a..ee41cd99 100644 --- a/docs/source/table_partitioning.rst +++ b/docs/source/table_partitioning.rst @@ -222,10 +222,31 @@ Command-line options ``--model-names`` ``-m`` ``None`` Optionally, a list of model names to partition for. ``--skip-create`` ``False`` Whether to skip creating partitions. ``--skip-delete`` ``False`` Whether to skip deleting partitions. + ``--detach`` ``'no'`` ``sequentially`` or ``concurrently`` to detach old partitions instead of deleting them. See below. ==================== ============= ================ ==================================================================================================== === === === === === === +Detaching instead of deleting +***************************** + +By default, partitions that are older than ``max_age`` are dropped together with their rows. Pass ``--detach`` to detach them from the partitioned table instead. The partition's table and its rows are kept as a regular table, so you can archive, export or re-attach it later. Detached tables are no longer partitions, so later runs of the command do not touch them. + +* ``--detach sequentially`` runs ``ALTER TABLE ... DETACH PARTITION ...`` in the same transaction that creates the new partitions. +* ``--detach concurrently`` runs ``ALTER TABLE ... DETACH PARTITION ... CONCURRENTLY`` (PostgreSQL 14 and newer). It only takes a ``SHARE UPDATE EXCLUSIVE`` lock on the partitioned table, so queries against it keep running. PostgreSQL does not allow this in a transaction, so new partitions are created in a transaction first and the old ones are then detached one by one. It cannot be used when the table has a default partition. If a concurrent detach is interrupted, PostgreSQL leaves the partition in a pending state; finish it with ``ALTER TABLE ... DETACH PARTITION ... FINALIZE``. + +The same is available from code: + +.. code-block:: python + + plan = manager.plan(skip_create=True) + plan.apply(detach=True) # or plan.apply(concurrently=True) + + # or for a single partition + with connection.schema_editor(atomic=False) as schema_editor: + schema_editor.detach_partition(MyModel, "2019_jan", concurrently=True) + + Configuration ************* diff --git a/psqlextra/backend/schema.py b/psqlextra/backend/schema.py index 1e4f2dda..32d5ae28 100644 --- a/psqlextra/backend/schema.py +++ b/psqlextra/backend/schema.py @@ -15,6 +15,7 @@ DatabaseSchemaEditor, ) from django.db.models import Field, Model +from django.db.transaction import TransactionManagementError from psqlextra.settings import ( postgres_prepend_local_search_path, @@ -85,6 +86,10 @@ class PostgresSchemaEditor(SchemaEditor): "CREATE TABLE %s PARTITION OF %s FOR VALUES IN (%s)" ) sql_delete_partition = "DROP TABLE %s" + sql_detach_partition = "ALTER TABLE %s DETACH PARTITION %s" + sql_detach_partition_concurrently = ( + "ALTER TABLE %s DETACH PARTITION %s CONCURRENTLY" + ) sql_table_comment = "COMMENT ON TABLE %s IS %s" side_effects: List[DatabaseSchemaEditor] = [ @@ -922,6 +927,55 @@ def delete_partition(self, model: Type[Model], name: str) -> None: ) self.execute(sql) + def detach_partition( + self, model: Type[Model], name: str, *, concurrently: bool = False + ) -> None: + """Detaches the partition with the specified name from the partitioned + table. + + Unlike :meth:`delete_partition`, the partition's table and the + rows in it are kept. The table becomes a regular, standalone + table that can be archived, exported or re-attached later. + + Arguments: + model: + Partitioned model to detach the partition from. + + name: + Name of the partition to detach. + + concurrently: + Whether to use ``DETACH PARTITION ... CONCURRENTLY`` + (PostgreSQL 14 and newer). This only takes a + ``SHARE UPDATE EXCLUSIVE`` lock on the partitioned table + instead of an ``ACCESS EXCLUSIVE`` lock, so queries + against the partitioned table keep running while the + partition is detached. + + PostgreSQL does not allow this inside a transaction + block. Use a schema editor created with + ``atomic=False`` outside of ``transaction.atomic()``. + """ + + if concurrently and self.connection.in_atomic_block: + raise TransactionManagementError( + "Partitions cannot be detached concurrently inside a " + "transaction. Use a schema editor with atomic=False " + "outside of transaction.atomic()." + ) + + template = ( + self.sql_detach_partition_concurrently + if concurrently + else self.sql_detach_partition + ) + + sql = template % ( + self.quote_name(model._meta.db_table), + self.quote_name(self.create_partition_table_name(model, name)), + ) + self.execute(sql) + def alter_db_table( self, model: Type[Model], old_db_table: str, new_db_table: str ) -> None: diff --git a/psqlextra/management/commands/pgpartition.py b/psqlextra/management/commands/pgpartition.py index ca621662..f3d65d33 100644 --- a/psqlextra/management/commands/pgpartition.py +++ b/psqlextra/management/commands/pgpartition.py @@ -65,6 +65,19 @@ def add_arguments(self, parser): default=False, ) + parser.add_argument( + "--detach", + choices=["no", "sequentially", "concurrently"], + help=( + "Detach old partitions instead of dropping them, keeping " + "their tables and rows. 'concurrently' uses DETACH PARTITION " + "... CONCURRENTLY (PostgreSQL 14+) so queries against the " + "partitioned table are not blocked." + ), + required=False, + default="no", + ) + def handle( # type: ignore[override] self, dry: bool, @@ -73,6 +86,7 @@ def handle( # type: ignore[override] skip_create: bool, skip_delete: bool, model_names: Optional[List[str]] = None, + detach: str = "no", *args, **kwargs, ): @@ -91,7 +105,8 @@ def handle( # type: ignore[override] print("Nothing to be done.") return - plan.print() + detach_partitions = detach != "no" + plan.print(detach=detach_partitions) if dry: return @@ -103,7 +118,11 @@ def handle( # type: ignore[override] print("Operation aborted.") return - plan.apply(using=using) + plan.apply( + using=using, + detach=detach_partitions, + concurrently=detach == "concurrently", + ) print("Operations applied.") @staticmethod diff --git a/psqlextra/partitioning/partition.py b/psqlextra/partitioning/partition.py index 4c13fda0..08af9d3c 100644 --- a/psqlextra/partitioning/partition.py +++ b/psqlextra/partitioning/partition.py @@ -29,6 +29,23 @@ def delete( ) -> None: """Deletes this partition from the database.""" + def detach( + self, + model: Type[PostgresPartitionedModel], + schema_editor: PostgresSchemaEditor, + *, + concurrently: bool = False, + ) -> None: + """Detaches this partition from the partitioned table, keeping the + partition's table and its rows. + + See :meth:`PostgresSchemaEditor.detach_partition`. + """ + + schema_editor.detach_partition( + model, self.name(), concurrently=concurrently + ) + def deconstruct(self) -> dict: """Deconstructs this partition into a dict of attributes/fields.""" diff --git a/psqlextra/partitioning/plan.py b/psqlextra/partitioning/plan.py index 301b4241..25efd39c 100644 --- a/psqlextra/partitioning/plan.py +++ b/psqlextra/partitioning/plan.py @@ -23,32 +23,69 @@ class PostgresModelPartitioningPlan: creations: List[PostgresPartition] = field(default_factory=list) deletions: List[PostgresPartition] = field(default_factory=list) - def apply(self, using: Optional[str]) -> None: - """Applies this partitioning plan by creating and deleting the planned - partitions. - - Applying the plan runs in a transaction. + def apply( + self, + using: Optional[str], + *, + detach: bool = False, + concurrently: bool = False, + ) -> None: + """Applies this partitioning plan by creating and deleting (or + detaching) the planned partitions. + + Applying the plan runs in a transaction. When detaching + concurrently, the partitions are created in a transaction first + and then detached one by one outside of it, because PostgreSQL + does not allow ``DETACH PARTITION ... CONCURRENTLY`` in a + transaction block. Arguments: using: Optional name of the database connection to use. + + detach: + Detach the partitions that would be deleted instead of + dropping them. Their tables and rows are kept. + + concurrently: + Detach with ``DETACH PARTITION ... CONCURRENTLY`` + (PostgreSQL 14 and newer) so that queries against the + partitioned table are not blocked. Implies ``detach``. """ connection = connections[using or "default"] + model = self.config.model + detach = detach or concurrently - with transaction.atomic(): + with transaction.atomic(using=using): with connection.schema_editor() as schema_editor: for partition in self.creations: partition.create( - self.config.model, + model, cast("PostgresSchemaEditor", schema_editor), comment=AUTO_PARTITIONED_COMMENT, ) + if not concurrently: + for partition in self.deletions: + if detach: + partition.detach( + model, + cast("PostgresSchemaEditor", schema_editor), + ) + else: + partition.delete( + model, + cast("PostgresSchemaEditor", schema_editor), + ) + + if concurrently and self.deletions: + with connection.schema_editor(atomic=False) as schema_editor: for partition in self.deletions: - partition.delete( - self.config.model, + partition.detach( + model, cast("PostgresSchemaEditor", schema_editor), + concurrently=True, ) def print(self) -> None: @@ -93,13 +130,25 @@ def deletions(self) -> List[PostgresPartition]: deletions.extend(model_plan.deletions) return deletions - def apply(self, using: Optional[str] = None) -> None: - """Applies this plan by creating/deleting all planned partitions.""" + def apply( + self, + using: Optional[str] = None, + *, + detach: bool = False, + concurrently: bool = False, + ) -> None: + """Applies this plan by creating/deleting all planned partitions. + + See :meth:`PostgresModelPartitioningPlan.apply` for ``detach`` + and ``concurrently``. + """ for model_plan in self.model_plans: - model_plan.apply(using=using) + model_plan.apply( + using=using, detach=detach, concurrently=concurrently + ) - def print(self) -> None: + def print(self, *, detach: bool = False) -> None: """Prints this plan to the terminal in a readable format.""" for model_plan in self.model_plans: @@ -109,7 +158,8 @@ def print(self) -> None: create_count = len(self.creations) delete_count = len(self.deletions) - print(f"{delete_count} partitions will be deleted") + action = "detached" if detach else "deleted" + print(f"{delete_count} partitions will be {action}") print(f"{create_count} partitions will be created") diff --git a/tests/test_management_command_partition.py b/tests/test_management_command_partition.py index c621cf15..20a7f11c 100644 --- a/tests/test_management_command_partition.py +++ b/tests/test_management_command_partition.py @@ -173,3 +173,45 @@ def test_management_command_partition_confirm_no( config.strategy.createable_partition.delete.assert_not_called() config.strategy.deleteable_partition.create.assert_not_called() config.strategy.deleteable_partition.delete.assert_not_called() + + +@pytest.mark.parametrize( + "mode,concurrently", + [("sequentially", False), ("concurrently", True)], + ids=["sequentially", "concurrently"], +) +def test_management_command_partition_detach( + mode, concurrently, run, fake_model, fake_partitioning_manager +): + """Tests whether the --detach option makes it detach partitions instead of + deleting them.""" + + config = fake_partitioning_manager.find_config_for_model(fake_model) + output = run("--yes", "--detach", mode) + + assert "1 partitions will be detached" in output + assert "Operations applied." in output + + config.strategy.createable_partition.create.assert_called_once() + config.strategy.deleteable_partition.delete.assert_not_called() + config.strategy.deleteable_partition.detach.assert_called_once() + assert ( + config.strategy.deleteable_partition.detach.call_args.kwargs.get( + "concurrently", False + ) + is concurrently + ) + + +def test_management_command_partition_detach_dry_run( + run, fake_model, fake_partitioning_manager +): + """Tests whether --dry together with --detach does not touch any + partition.""" + + config = fake_partitioning_manager.find_config_for_model(fake_model) + output = run("--dry", "--detach", "sequentially") + + assert "1 partitions will be detached" in output + config.strategy.deleteable_partition.detach.assert_not_called() + config.strategy.deleteable_partition.delete.assert_not_called() diff --git a/tests/test_partitioning_time.py b/tests/test_partitioning_time.py index 0ab0daf6..9a37d805 100644 --- a/tests/test_partitioning_time.py +++ b/tests/test_partitioning_time.py @@ -644,3 +644,99 @@ def test_partitioning_time_multiple_sizes(): with pytest.raises(PostgresPartitioningError): partition_by_current_time(model, weeks=1, months=2, count=1) + + +def _count_rows(table_name): + with connection.cursor() as cursor: + cursor.execute( + f"SELECT COUNT(*) FROM {connection.ops.quote_name(table_name)}" + ) + return cursor.fetchone()[0] + + +def _create_and_fill_monthly(model): + schema_editor = connection.schema_editor() + schema_editor.create_partitioned_model(model) + + manager = PostgresPartitioningManager( + [ + partition_by_current_time( + model, months=1, count=3, max_age=relativedelta(months=2) + ) + ] + ) + + with freezegun.freeze_time("2019-1-1"): + manager.plan().apply() + + model.objects.create(timestamp=datetime.datetime(2019, 1, 10)) + model.objects.create(timestamp=datetime.datetime(2019, 2, 10)) + model.objects.create(timestamp=datetime.datetime(2019, 3, 10)) + + return manager + + +@pytest.mark.postgres_version(lt=110000) +def test_partitioning_time_detach(): + """Tests whether partitions older than max_age are detached instead of + dropped, keeping their tables and rows.""" + + model = define_fake_partitioned_model( + {"timestamp": models.DateTimeField()}, {"key": ["timestamp"]} + ) + manager = _create_and_fill_monthly(model) + + with freezegun.freeze_time("2019-3-1"): + plan = manager.plan(skip_create=True) + assert [p.name() for p in plan.deletions] == ["2019_jan"] + plan.apply(detach=True) + + table = _get_partitioned_table(model) + assert [p.name for p in table.partitions] == ["2019_feb", "2019_mar"] + + detached_table_name = f"{model._meta.db_table}_2019_jan" + assert detached_table_name in db_introspection.table_names() + assert _count_rows(detached_table_name) == 1 + assert model.objects.count() == 2 + + # detached partitions are not partitions anymore, so running the plan + # again does not touch them + with freezegun.freeze_time("2019-3-1"): + assert manager.plan(skip_create=True).deletions == [] + + +@pytest.mark.postgres_version(lt=140000) +@pytest.mark.django_db(transaction=True) +def test_partitioning_time_detach_concurrently(): + """Tests whether partitions can be detached concurrently while new + partitions are created in the same plan.""" + + model = define_fake_partitioned_model( + {"timestamp": models.DateTimeField()}, {"key": ["timestamp"]} + ) + manager = _create_and_fill_monthly(model) + + with freezegun.freeze_time("2019-3-1"): + plan = manager.plan() + assert [p.name() for p in plan.creations] == ["2019_apr", "2019_may"] + assert [p.name() for p in plan.deletions] == ["2019_jan"] + plan.apply(concurrently=True) + + with transaction.atomic(): + table = _get_partitioned_table(model) + assert sorted(p.name for p in table.partitions) == [ + "2019_apr", + "2019_feb", + "2019_mar", + "2019_may", + ] + + detached_table_name = f"{model._meta.db_table}_2019_jan" + assert _count_rows(detached_table_name) == 1 + assert model.objects.count() == 2 + + with connection.schema_editor(atomic=False) as schema_editor: + schema_editor.delete_partitioned_model(model) + schema_editor.execute( + f"DROP TABLE {connection.ops.quote_name(detached_table_name)}" + ) diff --git a/tests/test_schema_editor_partitioning.py b/tests/test_schema_editor_partitioning.py index d3602c0d..c0fe2b9b 100644 --- a/tests/test_schema_editor_partitioning.py +++ b/tests/test_schema_editor_partitioning.py @@ -2,7 +2,8 @@ import pytest from django.core.exceptions import ImproperlyConfigured -from django.db import connection, models +from django.db import connection, models, transaction +from django.db.transaction import TransactionManagementError from psqlextra.backend.schema import PostgresSchemaEditor from psqlextra.types import PostgresPartitioningMethod @@ -411,3 +412,95 @@ def test_schema_editor_create_partitioned_custom_composite_primary_key( assert primary_key_constraint assert primary_key_constraint["columns"] == ["name", "timestamp"] + + +def _count_rows(table_name: str) -> int: + with connection.cursor() as cursor: + cursor.execute( + f"SELECT COUNT(*) FROM {connection.ops.quote_name(table_name)}" + ) + return cursor.fetchone()[0] + + +def _create_model_with_partitions(): + model = define_fake_partitioned_model( + {"name": models.TextField(), "timestamp": models.DateTimeField()}, + {"key": ["timestamp"]}, + ) + + schema_editor = PostgresSchemaEditor(connection) + schema_editor.create_partitioned_model(model) + schema_editor.add_range_partition( + model, name="jan", from_values="2019-1-1", to_values="2019-2-1" + ) + schema_editor.add_range_partition( + model, name="feb", from_values="2019-2-1", to_values="2019-3-1" + ) + + model.objects.create(name="a", timestamp="2019-01-15T00:00:00Z") + model.objects.create(name="b", timestamp="2019-01-20T00:00:00Z") + model.objects.create(name="c", timestamp="2019-02-15T00:00:00Z") + + return model + + +@pytest.mark.postgres_version(lt=110000) +def test_schema_editor_detach_partition(): + """Tests whether detaching a partition removes it from the partitioned + table but keeps the partition's table and rows.""" + + model = _create_model_with_partitions() + table_name = model._meta.db_table + partition_table_name = f"{table_name}_jan" + + schema_editor = PostgresSchemaEditor(connection) + schema_editor.detach_partition(model, "jan") + + table = db_introspection.get_partitioned_table(table_name) + assert [partition.name for partition in table.partitions] == ["feb"] + + assert partition_table_name in db_introspection.table_names() + assert _count_rows(partition_table_name) == 2 + assert model.objects.count() == 1 + + +@pytest.mark.postgres_version(lt=140000) +@pytest.mark.django_db(transaction=True) +def test_schema_editor_detach_partition_concurrently(): + """Tests whether a partition can be detached concurrently outside of a + transaction and whether its table and rows are kept.""" + + model = _create_model_with_partitions() + table_name = model._meta.db_table + partition_table_name = f"{table_name}_jan" + + with connection.schema_editor(atomic=False) as schema_editor: + schema_editor.detach_partition(model, "jan", concurrently=True) + + with transaction.atomic(): + table = db_introspection.get_partitioned_table(table_name) + assert [partition.name for partition in table.partitions] == ["feb"] + + assert _count_rows(partition_table_name) == 2 + assert model.objects.count() == 1 + + with connection.schema_editor(atomic=False) as schema_editor: + schema_editor.delete_partitioned_model(model) + schema_editor.execute( + f"DROP TABLE {connection.ops.quote_name(partition_table_name)}" + ) + + +@pytest.mark.postgres_version(lt=110000) +def test_schema_editor_detach_partition_concurrently_in_transaction(): + """Tests whether detaching concurrently inside a transaction raises a clear + error instead of failing in PostgreSQL.""" + + model = _create_model_with_partitions() + + schema_editor = PostgresSchemaEditor(connection) + with pytest.raises(TransactionManagementError): + schema_editor.detach_partition(model, "jan", concurrently=True) + + table = db_introspection.get_partitioned_table(model._meta.db_table) + assert len(table.partitions) == 2