Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions docs/source/table_partitioning.rst
Original file line number Diff line number Diff line change
Expand Up @@ -222,10 +222,31 @@ Command-line options
``--model-names`` ``-m`` ``None`` Optionally, a list of model names to partition for.
``--skip-create`` ``False`` Whether to skip creating partitions.
``--skip-delete`` ``False`` Whether to skip deleting partitions.
``--detach`` ``'no'`` ``sequentially`` or ``concurrently`` to detach old partitions instead of deleting them. See below.

==================== ============= ================ ==================================================================================================== === === === === === ===


Detaching instead of deleting
*****************************

By default, partitions that are older than ``max_age`` are dropped together with their rows. Pass ``--detach`` to detach them from the partitioned table instead. The partition's table and its rows are kept as a regular table, so you can archive, export or re-attach it later. Detached tables are no longer partitions, so later runs of the command do not touch them.

* ``--detach sequentially`` runs ``ALTER TABLE ... DETACH PARTITION ...`` in the same transaction that creates the new partitions.
* ``--detach concurrently`` runs ``ALTER TABLE ... DETACH PARTITION ... CONCURRENTLY`` (PostgreSQL 14 and newer). It only takes a ``SHARE UPDATE EXCLUSIVE`` lock on the partitioned table, so queries against it keep running. PostgreSQL does not allow this in a transaction, so new partitions are created in a transaction first and the old ones are then detached one by one. It cannot be used when the table has a default partition. If a concurrent detach is interrupted, PostgreSQL leaves the partition in a pending state; finish it with ``ALTER TABLE ... DETACH PARTITION ... FINALIZE``.

The same is available from code:

.. code-block:: python

plan = manager.plan(skip_create=True)
plan.apply(detach=True) # or plan.apply(concurrently=True)

# or for a single partition
with connection.schema_editor(atomic=False) as schema_editor:
schema_editor.detach_partition(MyModel, "2019_jan", concurrently=True)


Configuration
*************

Expand Down
54 changes: 54 additions & 0 deletions psqlextra/backend/schema.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
DatabaseSchemaEditor,
)
from django.db.models import Field, Model
from django.db.transaction import TransactionManagementError

from psqlextra.settings import (
postgres_prepend_local_search_path,
Expand Down Expand Up @@ -85,6 +86,10 @@ class PostgresSchemaEditor(SchemaEditor):
"CREATE TABLE %s PARTITION OF %s FOR VALUES IN (%s)"
)
sql_delete_partition = "DROP TABLE %s"
sql_detach_partition = "ALTER TABLE %s DETACH PARTITION %s"
sql_detach_partition_concurrently = (
"ALTER TABLE %s DETACH PARTITION %s CONCURRENTLY"
)
sql_table_comment = "COMMENT ON TABLE %s IS %s"

side_effects: List[DatabaseSchemaEditor] = [
Expand Down Expand Up @@ -922,6 +927,55 @@ def delete_partition(self, model: Type[Model], name: str) -> None:
)
self.execute(sql)

def detach_partition(
self, model: Type[Model], name: str, *, concurrently: bool = False
) -> None:
"""Detaches the partition with the specified name from the partitioned
table.

Unlike :meth:`delete_partition`, the partition's table and the
rows in it are kept. The table becomes a regular, standalone
table that can be archived, exported or re-attached later.

Arguments:
model:
Partitioned model to detach the partition from.

name:
Name of the partition to detach.

concurrently:
Whether to use ``DETACH PARTITION ... CONCURRENTLY``
(PostgreSQL 14 and newer). This only takes a
``SHARE UPDATE EXCLUSIVE`` lock on the partitioned table
instead of an ``ACCESS EXCLUSIVE`` lock, so queries
against the partitioned table keep running while the
partition is detached.

PostgreSQL does not allow this inside a transaction
block. Use a schema editor created with
``atomic=False`` outside of ``transaction.atomic()``.
"""

if concurrently and self.connection.in_atomic_block:
raise TransactionManagementError(
"Partitions cannot be detached concurrently inside a "
"transaction. Use a schema editor with atomic=False "
"outside of transaction.atomic()."
)

template = (
self.sql_detach_partition_concurrently
if concurrently
else self.sql_detach_partition
)

sql = template % (
self.quote_name(model._meta.db_table),
self.quote_name(self.create_partition_table_name(model, name)),
)
self.execute(sql)

def alter_db_table(
self, model: Type[Model], old_db_table: str, new_db_table: str
) -> None:
Expand Down
23 changes: 21 additions & 2 deletions psqlextra/management/commands/pgpartition.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,19 @@ def add_arguments(self, parser):
default=False,
)

parser.add_argument(
"--detach",
choices=["no", "sequentially", "concurrently"],
help=(
"Detach old partitions instead of dropping them, keeping "
"their tables and rows. 'concurrently' uses DETACH PARTITION "
"... CONCURRENTLY (PostgreSQL 14+) so queries against the "
"partitioned table are not blocked."
),
required=False,
default="no",
)

def handle( # type: ignore[override]
self,
dry: bool,
Expand All @@ -73,6 +86,7 @@ def handle( # type: ignore[override]
skip_create: bool,
skip_delete: bool,
model_names: Optional[List[str]] = None,
detach: str = "no",
*args,
**kwargs,
):
Expand All @@ -91,7 +105,8 @@ def handle( # type: ignore[override]
print("Nothing to be done.")
return

plan.print()
detach_partitions = detach != "no"
plan.print(detach=detach_partitions)

if dry:
return
Expand All @@ -103,7 +118,11 @@ def handle( # type: ignore[override]
print("Operation aborted.")
return

plan.apply(using=using)
plan.apply(
using=using,
detach=detach_partitions,
concurrently=detach == "concurrently",
)
print("Operations applied.")

@staticmethod
Expand Down
17 changes: 17 additions & 0 deletions psqlextra/partitioning/partition.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,23 @@ def delete(
) -> None:
"""Deletes this partition from the database."""

def detach(
self,
model: Type[PostgresPartitionedModel],
schema_editor: PostgresSchemaEditor,
*,
concurrently: bool = False,
) -> None:
"""Detaches this partition from the partitioned table, keeping the
partition's table and its rows.

See :meth:`PostgresSchemaEditor.detach_partition`.
"""

schema_editor.detach_partition(
model, self.name(), concurrently=concurrently
)

def deconstruct(self) -> dict:
"""Deconstructs this partition into a dict of attributes/fields."""

Expand Down
78 changes: 64 additions & 14 deletions psqlextra/partitioning/plan.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,32 +23,69 @@ class PostgresModelPartitioningPlan:
creations: List[PostgresPartition] = field(default_factory=list)
deletions: List[PostgresPartition] = field(default_factory=list)

def apply(self, using: Optional[str]) -> None:
"""Applies this partitioning plan by creating and deleting the planned
partitions.

Applying the plan runs in a transaction.
def apply(
self,
using: Optional[str],
*,
detach: bool = False,
concurrently: bool = False,
) -> None:
"""Applies this partitioning plan by creating and deleting (or
detaching) the planned partitions.

Applying the plan runs in a transaction. When detaching
concurrently, the partitions are created in a transaction first
and then detached one by one outside of it, because PostgreSQL
does not allow ``DETACH PARTITION ... CONCURRENTLY`` in a
transaction block.

Arguments:
using:
Optional name of the database connection to use.

detach:
Detach the partitions that would be deleted instead of
dropping them. Their tables and rows are kept.

concurrently:
Detach with ``DETACH PARTITION ... CONCURRENTLY``
(PostgreSQL 14 and newer) so that queries against the
partitioned table are not blocked. Implies ``detach``.
"""

connection = connections[using or "default"]
model = self.config.model
detach = detach or concurrently

with transaction.atomic():
with transaction.atomic(using=using):
with connection.schema_editor() as schema_editor:
for partition in self.creations:
partition.create(
self.config.model,
model,
cast("PostgresSchemaEditor", schema_editor),
comment=AUTO_PARTITIONED_COMMENT,
)

if not concurrently:
for partition in self.deletions:
if detach:
partition.detach(
model,
cast("PostgresSchemaEditor", schema_editor),
)
else:
partition.delete(
model,
cast("PostgresSchemaEditor", schema_editor),
)

if concurrently and self.deletions:
with connection.schema_editor(atomic=False) as schema_editor:
for partition in self.deletions:
partition.delete(
self.config.model,
partition.detach(
model,
cast("PostgresSchemaEditor", schema_editor),
concurrently=True,
)

def print(self) -> None:
Expand Down Expand Up @@ -93,13 +130,25 @@ def deletions(self) -> List[PostgresPartition]:
deletions.extend(model_plan.deletions)
return deletions

def apply(self, using: Optional[str] = None) -> None:
"""Applies this plan by creating/deleting all planned partitions."""
def apply(
self,
using: Optional[str] = None,
*,
detach: bool = False,
concurrently: bool = False,
) -> None:
"""Applies this plan by creating/deleting all planned partitions.

See :meth:`PostgresModelPartitioningPlan.apply` for ``detach``
and ``concurrently``.
"""

for model_plan in self.model_plans:
model_plan.apply(using=using)
model_plan.apply(
using=using, detach=detach, concurrently=concurrently
)

def print(self) -> None:
def print(self, *, detach: bool = False) -> None:
"""Prints this plan to the terminal in a readable format."""

for model_plan in self.model_plans:
Expand All @@ -109,7 +158,8 @@ def print(self) -> None:
create_count = len(self.creations)
delete_count = len(self.deletions)

print(f"{delete_count} partitions will be deleted")
action = "detached" if detach else "deleted"
print(f"{delete_count} partitions will be {action}")
print(f"{create_count} partitions will be created")


Expand Down
42 changes: 42 additions & 0 deletions tests/test_management_command_partition.py
Original file line number Diff line number Diff line change
Expand Up @@ -173,3 +173,45 @@ def test_management_command_partition_confirm_no(
config.strategy.createable_partition.delete.assert_not_called()
config.strategy.deleteable_partition.create.assert_not_called()
config.strategy.deleteable_partition.delete.assert_not_called()


@pytest.mark.parametrize(
"mode,concurrently",
[("sequentially", False), ("concurrently", True)],
ids=["sequentially", "concurrently"],
)
def test_management_command_partition_detach(
mode, concurrently, run, fake_model, fake_partitioning_manager
):
"""Tests whether the --detach option makes it detach partitions instead of
deleting them."""

config = fake_partitioning_manager.find_config_for_model(fake_model)
output = run("--yes", "--detach", mode)

assert "1 partitions will be detached" in output
assert "Operations applied." in output

config.strategy.createable_partition.create.assert_called_once()
config.strategy.deleteable_partition.delete.assert_not_called()
config.strategy.deleteable_partition.detach.assert_called_once()
assert (
config.strategy.deleteable_partition.detach.call_args.kwargs.get(
"concurrently", False
)
is concurrently
)


def test_management_command_partition_detach_dry_run(
run, fake_model, fake_partitioning_manager
):
"""Tests whether --dry together with --detach does not touch any
partition."""

config = fake_partitioning_manager.find_config_for_model(fake_model)
output = run("--dry", "--detach", "sequentially")

assert "1 partitions will be detached" in output
config.strategy.deleteable_partition.detach.assert_not_called()
config.strategy.deleteable_partition.delete.assert_not_called()
Loading