Coverage for app/tasks.py: 84.34%
728 statements
« prev ^ index » next coverage.py v7.15.2, created at 2026-07-25 16:05 +0000
1import json
2import os
3import re
4from typing import Any
6import git
8from .celery_app import celery_app
9from .config import settings
10from .services import (
11 OpenStackService,
12 PackerBuildLock,
13 PackerExecutor,
14 PerTaskCloudsConfig,
15 TerraformExecutor,
16 git_service,
17)
18from .services.packer_discovery import PackerTemplateDiscoveryError, _discover_packer_templates, _PackerTemplate
19from .utils.logger import LogCategory, get_logger
21logger = get_logger(__name__)
24def _tfstate_schema_name(deployment_id: str) -> str:
25 """Postgres schema name for one deployment's Terraform state.
27 UUIDs contain hyphens, which would force every reference to be
28 double-quoted. Replacing hyphens with underscores keeps the schema
29 a plain unquoted identifier and avoids escaping hazards in any
30 backend-config plumbing.
31 """
32 return f"deployment_{deployment_id.replace('-', '_')}"
35class Failure(Exception):
36 """Custom exception that carries deployment details for Celery.
38 The full failure payload is serialised once into ``args[0]`` as a JSON
39 string. The backend's celery event listener parses that JSON back via
40 a ``Failure\\('<json>'\\)`` regex over the traceback.
42 ``__reduce__`` is overridden so pickle reconstructs the exception via
43 the ``_from_payload`` classmethod, which accepts the single JSON string
44 directly.
45 """
47 def __init__(
48 self,
49 message: str,
50 deployment_id: str,
51 logs_dict: list[dict[str, Any]] | dict[str, Any],
52 tf_state: str | None = None,
53 commit_info: dict[str, Any] | None = None,
54 terraform_outputs: dict[str, Any] | None = None,
55 ):
56 self.deployment_id = deployment_id
57 self.logs_dict = logs_dict
58 self.tf_state = tf_state
59 self.commit_info = commit_info
60 self.terraform_outputs = terraform_outputs
62 # Encode all data as JSON in the exception message
63 data = {
64 "error": message,
65 "deployment_id": deployment_id,
66 "logs": logs_dict,
67 "tf_state": tf_state,
68 "commit_info": commit_info,
69 "terraform_outputs": terraform_outputs,
70 }
71 super().__init__(json.dumps(data))
73 @classmethod
74 def _from_payload(cls, payload: str) -> "Failure":
75 """Reconstruct a Failure from its serialised JSON payload.
77 Used by ``__reduce__`` so pickle can round-trip the exception.
78 """
79 data = json.loads(payload)
80 instance = cls.__new__(cls)
81 instance.deployment_id = data.get("deployment_id", "")
82 instance.logs_dict = data.get("logs")
83 instance.tf_state = data.get("tf_state")
84 instance.commit_info = data.get("commit_info")
85 instance.terraform_outputs = data.get("terraform_outputs")
86 Exception.__init__(instance, payload)
87 return instance
89 def __reduce__(self):
90 # The single-arg constructor here is ``_from_payload``; args[0] is
91 # the JSON string we built in __init__.
92 return (Failure._from_payload, (self.args[0] if self.args else "{}",))
94 def __repr__(self) -> str:
95 # Pin the repr format that the backend's celery event listener
96 # relies on (regex ``Failure\('(.+)'\)``).
97 return f"Failure({self.args[0]!r})" if self.args else "Failure()"
99 def to_dict(self) -> dict[str, Any]:
100 """Convert exception data to dict for serialization"""
101 return json.loads(str(self))
104# --- Variable encoding for Packer/Terraform CLI ----------------------------
107def _looks_like_file_var_value(value: Any) -> bool:
108 """True if ``value`` matches the file-upload shape produced by
109 the backend's ``_attach_files_to_user_input``: a non-empty
110 mapping whose entries each carry a ``content_b64`` field plus
111 the metadata triplet (name, size, content_type) — i.e. exactly
112 the ``map(object(...))`` HCL contract.
114 Used by :func:`_strip_file_vars` so destroy / cleanup-after-
115 failure can drop ``@openstack:file:*``-marked variables before
116 passing the var-set to ``terraform destroy``. Terraform
117 validates *all* declared variables on every command — including
118 destroy — so an apply-only file-var would otherwise block the
119 cleanup with a schema error.
121 A slot must carry ``content_b64`` to qualify as a file-var; the
122 strictness avoids dropping legitimate non-file map variables that
123 happen to share the metadata keys.
124 """
125 if not isinstance(value, dict) or not value:
126 return False
127 for slot in value.values():
128 if not isinstance(slot, dict):
129 return False
130 if "content_b64" not in slot:
131 return False
132 return True
135def _strip_file_vars(terraform_vars: dict[str, Any]) -> dict[str, Any]:
136 """Return a copy of ``terraform_vars`` with file-shape entries removed.
138 Pure function — never mutates the input. Used by destroy and the
139 deploy cleanup-after-failure branches; deploy itself keeps the
140 file vars because ``apply`` consumes them via cloud-init.
141 """
142 return {k: v for k, v in terraform_vars.items() if not _looks_like_file_var_value(v)}
145def _scrub_nested_nones(value: Any) -> Any:
146 """Recursively drop ``None`` entries from nested dicts/lists.
148 A stray ``None`` inside a ``map(list(string))`` slot would surface as
149 literal HCL ``null`` after the JSON round-trip and trip Terraform's
150 type check. Dicts have their ``None``-valued keys removed, lists have
151 their ``None`` entries filtered out, and both are walked recursively.
152 Scalars (including bools) pass through untouched.
153 """
154 if isinstance(value, dict):
155 cleaned: dict[Any, Any] = {}
156 for k, v in value.items():
157 if v is None:
158 continue
159 cleaned[k] = _scrub_nested_nones(v)
160 return cleaned
161 if isinstance(value, list):
162 return [_scrub_nested_nones(item) for item in value if item is not None]
163 return value
166def encode_terraform_vars(d: dict[str, Any]) -> dict[str, str]:
167 """Encode variables for ``terraform -var key=value`` CLI args.
169 Terraform reads complex types (objects, tuples) when the value is a
170 valid JSON literal. We JSON-encode dicts/lists once and pass them
171 through verbatim — no string normalisation that could damage escape
172 sequences.
174 Nested ``None`` values are scrubbed recursively (see
175 :func:`_scrub_nested_nones`) so a stray ``null`` deep inside a
176 ``map(list(string))`` slot can't trip Terraform's type check.
177 """
178 result: dict[str, str] = {}
179 for k, v in d.items():
180 if v is None:
181 continue
182 if isinstance(v, bool):
183 # HCL accepts lowercase only; ``str(True)`` would emit "True".
184 result[k] = "true" if v else "false"
185 elif isinstance(v, dict | list):
186 result[k] = json.dumps(_scrub_nested_nones(v), ensure_ascii=False)
187 else:
188 result[k] = str(v)
189 return result
192def encode_packer_vars(d: dict[str, Any]) -> dict[str, str]:
193 """Encode variables for ``packer -var key=value`` CLI args.
195 For HCL ``list(...)``-typed variables, we emit a JSON array literal
196 (e.g. ``["NAT"]``) — that's the only form Packer accepts via ``-var``
197 for typed-list variables, since Packer parses each ``-var`` value as
198 an HCL expression against the declared type. JSON arrays are valid
199 HCL list literals, so a single representation covers both syntaxes.
200 String values are passed through verbatim.
201 """
202 result: dict[str, str] = {}
203 for k, v in d.items():
204 if v is None:
205 continue
206 if isinstance(v, list):
207 # JSON array works for ``list(string)``, ``list(number)`` etc.
208 # ``ensure_ascii=False`` lets non-ASCII names pass through
209 # unchanged (Packer's HCL parser is UTF-8 native).
210 result[k] = json.dumps(v, ensure_ascii=False)
211 elif isinstance(v, dict):
212 # ``map(...)``-typed Packer vars take the same JSON literal
213 # path. No Packer template in the project uses this today,
214 # but the encoding is correct for when one shows up.
215 result[k] = json.dumps(v, ensure_ascii=False)
216 elif isinstance(v, bool):
217 result[k] = "true" if v else "false"
218 else:
219 result[k] = str(v)
220 return result
223# --- Phase tracking ----------------------------------------------------------
224#
225# Phases are pinned by name (a string the frontend renders as a stepper) and
226# by index (1-based, used for the percent bar). The list is split in two so
227# the worker can collapse the Packer block when a deployment doesn't need a
228# Packer build — that decision is made after the git clone has finished and
229# we can see whether ``packer/template.pkr.hcl`` exists.
231PHASE_STARTING = "STARTING"
232PHASE_OPENSTACK_SETUP = "OPENSTACK_SETUP"
233PHASE_GIT_CLONE = "GIT_CLONE"
234PHASE_CREDS_MATERIALISE = "CREDS_MATERIALISE"
235PHASE_PACKER_INIT = "PACKER_INIT"
236PHASE_PACKER_VALIDATE = "PACKER_VALIDATE"
237PHASE_PACKER_BUILD = "PACKER_BUILD"
238PHASE_TERRAFORM_INIT = "TERRAFORM_INIT"
239PHASE_TERRAFORM_PLAN = "TERRAFORM_PLAN"
240PHASE_TERRAFORM_APPLY = "TERRAFORM_APPLY"
241PHASE_OUTPUTS_AND_CLEANUP = "OUTPUTS_AND_CLEANUP"
242PHASE_TERRAFORM_DESTROY = "TERRAFORM_DESTROY"
243PHASE_CLEANUP = "CLEANUP"
244# Pause/resume share the deploy/destroy preamble but their hot phase is
245# a CLI-driven server stop/start, so they get distinct phase names rather
246# than reusing TERRAFORM_DESTROY.
247PHASE_SERVER_STOP = "SERVER_STOP"
248PHASE_SERVER_START = "SERVER_START"
250_PHASES_WITH_PACKER = (
251 PHASE_STARTING,
252 PHASE_OPENSTACK_SETUP,
253 PHASE_GIT_CLONE,
254 PHASE_CREDS_MATERIALISE,
255 PHASE_PACKER_INIT,
256 PHASE_PACKER_VALIDATE,
257 PHASE_PACKER_BUILD,
258 PHASE_TERRAFORM_INIT,
259 PHASE_TERRAFORM_PLAN,
260 PHASE_TERRAFORM_APPLY,
261 PHASE_OUTPUTS_AND_CLEANUP,
262)
263_PHASES_WITHOUT_PACKER = (
264 PHASE_STARTING,
265 PHASE_OPENSTACK_SETUP,
266 PHASE_GIT_CLONE,
267 PHASE_CREDS_MATERIALISE,
268 PHASE_TERRAFORM_INIT,
269 PHASE_TERRAFORM_PLAN,
270 PHASE_TERRAFORM_APPLY,
271 PHASE_OUTPUTS_AND_CLEANUP,
272)
275def _phases_for_templates(templates: list[_PackerTemplate]) -> tuple[str, ...]:
276 """Build the phase tuple based on the discovered Packer templates.
278 * No templates → ``_PHASES_WITHOUT_PACKER`` (clone, then straight
279 to terraform).
280 * One template with key ``"default"`` (legacy layout) →
281 ``_PHASES_WITH_PACKER`` verbatim. The phase names stay
282 ``PACKER_INIT`` / ``PACKER_VALIDATE`` / ``PACKER_BUILD`` with no
283 key suffix.
284 * Multi (any other shape) → one
285 ``PACKER_INIT:<key>`` / ``PACKER_VALIDATE:<key>`` /
286 ``PACKER_BUILD:<key>`` trio per template, inserted where the Packer
287 phases sit in ``_PHASES_WITH_PACKER``. Templates are emitted in the
288 order passed in (discovery returns them sorted by key, so the
289 stepper order is deterministic).
290 """
291 if not templates:
292 return _PHASES_WITHOUT_PACKER
293 if len(templates) == 1 and templates[0].key == "default":
294 return _PHASES_WITH_PACKER
296 idx = next(
297 (i for i, p in enumerate(_PHASES_WITH_PACKER) if p == PHASE_PACKER_INIT),
298 0,
299 )
300 prefix = tuple(_PHASES_WITH_PACKER[:idx])
301 suffix = tuple(
302 p for p in _PHASES_WITH_PACKER[idx:] if p not in (PHASE_PACKER_INIT, PHASE_PACKER_VALIDATE, PHASE_PACKER_BUILD)
303 )
304 packer_phases: list[str] = []
305 for t in templates:
306 packer_phases.extend(
307 [
308 f"{PHASE_PACKER_INIT}:{t.key}",
309 f"{PHASE_PACKER_VALIDATE}:{t.key}",
310 f"{PHASE_PACKER_BUILD}:{t.key}",
311 ]
312 )
313 return prefix + tuple(packer_phases) + suffix
316# Destroy uses a shorter pipeline — no Packer (we don't need a fresh
317# image to tear things down) and no plan (terraform destroy has its own
318# planning step internally that we don't surface as its own progress
319# phase).
320_PHASES_DESTROY = (
321 PHASE_STARTING,
322 PHASE_OPENSTACK_SETUP,
323 PHASE_GIT_CLONE,
324 PHASE_CREDS_MATERIALISE,
325 PHASE_TERRAFORM_INIT,
326 PHASE_TERRAFORM_DESTROY,
327 PHASE_CLEANUP,
328)
329# Per-VM redeploy reuses the destroy preamble (git clone at the same
330# release tag, materialise clouds.yaml, terraform init) and then runs
331# ``terraform apply -replace=<addr> -target=<addr>`` instead of
332# ``destroy``. The shape mirrors destroy exactly so the SSE phase bar
333# in the UI stays familiar.
334_PHASES_REDEPLOY = (
335 PHASE_STARTING,
336 PHASE_OPENSTACK_SETUP,
337 PHASE_GIT_CLONE,
338 PHASE_CREDS_MATERIALISE,
339 PHASE_TERRAFORM_INIT,
340 PHASE_TERRAFORM_APPLY,
341 PHASE_CLEANUP,
342)
343# Pause / resume share the destroy preamble — git clone at the same
344# release tag, materialise clouds.yaml, terraform init so we can pull
345# the canonical state from the pg backend. The hot phase is the
346# server stop / start loop. CLEANUP runs the repo shred and (for the
347# log) a final state pull, mirroring destroy's tail.
348_PHASES_PAUSE = (
349 PHASE_STARTING,
350 PHASE_OPENSTACK_SETUP,
351 PHASE_GIT_CLONE,
352 PHASE_CREDS_MATERIALISE,
353 PHASE_TERRAFORM_INIT,
354 PHASE_SERVER_STOP,
355 PHASE_CLEANUP,
356)
357_PHASES_RESUME = (
358 PHASE_STARTING,
359 PHASE_OPENSTACK_SETUP,
360 PHASE_GIT_CLONE,
361 PHASE_CREDS_MATERIALISE,
362 PHASE_TERRAFORM_INIT,
363 PHASE_SERVER_START,
364 PHASE_CLEANUP,
365)
368class _PhaseTracker:
369 """Drives ``StructuredLogger.progress`` calls.
371 The set of phases is fixed at construction time so the percent bar
372 monotonically advances; ``mark()`` looks up the index of the named
373 phase and sends a progress event with the correct ``idx/total``.
374 """
376 def __init__(self, logger: Any, phases: tuple[str, ...]):
377 self._logger = logger
378 self._phases = phases
379 self._index_by_name = {name: i for i, name in enumerate(phases, start=1)}
381 @property
382 def total(self) -> int:
383 return len(self._phases)
385 def mark(self, phase_name: str, message: str = "") -> None:
386 idx = self._index_by_name.get(phase_name)
387 if idx is None:
388 # Unknown phase — emit a transcript marker but no progress
389 # update so the bar doesn't reset.
390 self._logger.phase(phase_name)
391 return
392 # Buffer the readable phase header and emit the live progress event.
393 # The full phase-name sequence rides on every event so the UI can
394 # render each stepper slot with its real label immediately.
395 self._logger.phase(phase_name)
396 self._logger.progress(
397 phase_name,
398 idx,
399 self.total,
400 message,
401 phase_names=self._phases,
402 )
405def _terraform_executor(terraform_dir, openstack_env, tfstate_conn_str, tfstate_schema) -> TerraformExecutor:
406 """Build a TerraformExecutor bound to the deployment's pg-backend schema."""
407 return TerraformExecutor(
408 terraform_dir,
409 env_vars=openstack_env,
410 backend_conn_str=tfstate_conn_str,
411 backend_schema_name=tfstate_schema,
412 )
415def collect_terraform_state_helper(
416 terraform_dir, openstack_env, tfstate_conn_str, tfstate_schema, task_logger, *, local_fallback=False
417):
418 """Snapshot the terraform state for the task row (best-effort).
420 With the pg backend the canonical state lives in Postgres; this
421 snapshot is used for debugging only. When ``local_fallback`` is set
422 (deploy path), a missing/empty pull falls back to reading the local
423 ``terraform.tfstate`` file for legacy/test modes that don't configure
424 a remote backend. Returns ``None`` when nothing could be read.
425 """
426 if not (terraform_dir and os.path.exists(terraform_dir)):
427 return None
428 try:
429 pulled = _terraform_executor(terraform_dir, openstack_env, tfstate_conn_str, tfstate_schema).state_pull()
430 if pulled or not local_fallback:
431 return pulled
432 except Exception as e:
433 task_logger.warning(f"Could not pull terraform state: {e}", category=LogCategory.WARNING)
434 if not local_fallback:
435 return None
437 # Legacy fallback — only relevant when no pg backend is configured.
438 tfstate_path = os.path.join(terraform_dir, "terraform.tfstate")
439 if os.path.exists(tfstate_path):
440 try:
441 with open(tfstate_path) as f:
442 return f.read()
443 except Exception as e:
444 task_logger.warning(f"Could not read terraform state: {e}", category=LogCategory.WARNING)
445 return None
448def collect_terraform_outputs_helper(terraform_dir, openstack_env, tfstate_conn_str, tfstate_schema, task_logger):
449 """Collect terraform outputs even on partial success. ``None`` on failure."""
450 if terraform_dir and os.path.exists(terraform_dir):
451 try:
452 return _terraform_executor(terraform_dir, openstack_env, tfstate_conn_str, tfstate_schema).output()
453 except Exception as e:
454 task_logger.warning(f"Could not read terraform outputs: {e}", category=LogCategory.WARNING)
455 return None
458def _extract_commit_info(repo_path: str) -> dict[str, Any]:
459 """Read the checked-out commit's metadata from a cloned repo.
461 Returns the dict shape persisted into the task result and the
462 ``Failure`` payload. Callers wrap this in their own try/except so a
463 repo without a readable HEAD degrades to a warning, not a hard fail.
464 """
465 repo = git.Repo(repo_path)
466 commit = repo.head.commit
467 return {
468 "hash": commit.hexsha,
469 "message": commit.message.strip(),
470 "author": str(commit.author),
471 "date": commit.committed_datetime.isoformat(),
472 }
475def _build_image_names(templates: list[_PackerTemplate], app_id: str, image_tag: str) -> dict[str, str]:
476 """Reconstruct the per-template Glance image-name map.
478 Legacy single-template apps (or apps with no Packer at all) keep the
479 flat ``{"default": "<app_id>-<tag>"}`` shape; multi-image apps get one
480 ``<app_id>-<key>-<tag>`` entry per template. Used by destroy/redeploy,
481 which must name the same images the original deploy built so
482 Terraform's variable validation matches the pg-backend state.
483 """
484 if not templates or (len(templates) == 1 and templates[0].key == "default"):
485 return {"default": f"{app_id}-{image_tag}"}
486 return {t.key: f"{app_id}-{t.key}-{image_tag}" for t in templates}
489def _apply_image_name_vars(target: dict[str, Any], image_names: dict[str, str], *, legacy: bool) -> None:
490 """Inject the image-name variable(s) into a Terraform var-set.
492 Legacy layout gets a single flat ``image_name``; multi-image apps get
493 one ``image_name_<key>`` per template. ``legacy`` is decided by the
494 caller so this stays a pure mapping of the existing branch bodies.
495 """
496 if legacy:
497 target["image_name"] = image_names["default"]
498 else:
499 for key, name in image_names.items():
500 target[f"image_name_{key}"] = name
503def _cleanup_task_resources(clouds_config: PerTaskCloudsConfig | None, repo_path: str | None, task_logger: Any) -> None:
504 """Best-effort teardown shared by every task's ``finally`` block.
506 Shreds the per-task clouds.yaml first (so the credential file is gone
507 even if the repo cleanup below fails or hangs), then removes the
508 cloned repo. Both steps swallow their own errors as warnings so the
509 task's real result/exception is never masked by cleanup noise.
510 """
511 if clouds_config is not None:
512 try:
513 clouds_config.__exit__(None, None, None)
514 except Exception as e:
515 task_logger.warning(
516 f"Per-task clouds.yaml cleanup failed: {e}",
517 category=LogCategory.WARNING,
518 )
519 if repo_path:
520 try:
521 git_service.cleanup_repository(repo_path)
522 task_logger.success("Repository cleanup completed", category=LogCategory.SYSTEM)
523 except Exception as e:
524 task_logger.warning(f"Repository cleanup failed: {e}", category=LogCategory.WARNING)
527def _build_one_packer_image(
528 tmpl,
529 *,
530 image_name,
531 is_legacy,
532 openstack_service,
533 project_id,
534 repo_path,
535 openstack_env,
536 stream_line,
537 user_vars,
538 phase_tracker,
539 task_logger,
540):
541 """Build (or reuse) the Packer image for a single template.
543 Skips the build when the image already exists in Glance, and
544 coordinates concurrent workers via ``PackerBuildLock`` (only one
545 worker builds a given image; the others wait and reuse it). Raises
546 ``Exception("Packer error: ...")`` on any failure.
547 """
548 log_prefix = "" if is_legacy else f"[{tmpl.key}] "
550 # Phase names: legacy stays unsuffixed; multi-template apps get one
551 # ``PHASE:<key>`` trio per template.
552 init_phase = PHASE_PACKER_INIT if is_legacy else f"{PHASE_PACKER_INIT}:{tmpl.key}"
553 validate_phase = PHASE_PACKER_VALIDATE if is_legacy else f"{PHASE_PACKER_VALIDATE}:{tmpl.key}"
554 build_phase = PHASE_PACKER_BUILD if is_legacy else f"{PHASE_PACKER_BUILD}:{tmpl.key}"
556 build_lock = PackerBuildLock(project_id, image_name)
557 wait_announced = False
558 try:
559 while True:
560 # If the image already exists, skip the build and the lock.
561 exists, image_id = openstack_service.check_image_exists(image_name)
562 if exists:
563 task_logger.success(
564 f"{log_prefix}Image '{image_name}' already exists (ID: {image_id}). Skipping Packer build.",
565 category=LogCategory.STATUS,
566 )
567 break
569 held = build_lock.acquire_or_wait()
570 if not held:
571 # Another worker is still building the same image. Surface
572 # this in the per-deployment log once so the frontend's
573 # live tail shows *something* during the 5-second poll
574 # cycles — without it the browser sees no events and looks
575 # frozen.
576 if not wait_announced:
577 task_logger.info(
578 f"{log_prefix}Another worker is currently building image '{image_name}'. Waiting…",
579 category=LogCategory.STATUS,
580 )
581 wait_announced = True
582 # We slept inside acquire_or_wait; re-check Glance.
583 continue
585 # Re-check after acquiring: another worker may have finished its
586 # build between our last check and our lock acquisition.
587 exists, image_id = openstack_service.check_image_exists(image_name)
588 if exists:
589 task_logger.success(
590 f"{log_prefix}Image '{image_name}' built by another worker (ID: {image_id}). Skipping.",
591 category=LogCategory.STATUS,
592 )
593 break
595 task_logger.info(
596 f"{log_prefix}Image '{image_name}' does not exist. Building...",
597 category=LogCategory.OPERATION,
598 )
600 # Pick the right packer working directory: legacy uses
601 # ``packer/`` directly; multi uses ``packer/<key>/``. Template
602 # file name is always ``template.pkr.hcl`` relative to that
603 # directory.
604 packer_dir = os.path.join(repo_path, "packer") if is_legacy else os.path.join(repo_path, "packer", tmpl.key)
605 packer = PackerExecutor(
606 packer_dir,
607 env_vars=openstack_env,
608 output_callback=stream_line,
609 )
611 # Per-template Packer variables. Legacy shape is the flat
612 # ``user_vars["packer"][var_name]``; multi shape is nested
613 # ``user_vars["packer"][template_key][var_name]``.
614 if is_legacy:
615 user_packer = user_vars.get("packer", {})
616 else:
617 user_packer = (user_vars.get("packer") or {}).get(tmpl.key, {}) or {}
618 packer_vars = {**user_packer}
619 packer_vars["image_name"] = image_name
620 packer_vars = encode_packer_vars(packer_vars)
622 task_logger.info(
623 f"{log_prefix}Packer variable keys",
624 category=LogCategory.OPERATION,
625 keys=list(packer_vars.keys()),
626 template=tmpl.key,
627 image_name=image_name,
628 )
630 phase_tracker.mark(init_phase, f"{log_prefix}Initializing Packer plugins")
631 success, stdout, stderr = packer.init()
632 if not success:
633 if stdout:
634 task_logger.command_output("packer_init_stdout", stdout, returncode=1)
635 if stderr:
636 task_logger.command_output("packer_init_stderr", stderr, returncode=1)
637 raise Exception(f"{log_prefix}Packer init failed")
639 phase_tracker.mark(validate_phase, f"{log_prefix}Validating Packer template")
640 success, stdout, stderr = packer.validate("template.pkr.hcl", packer_vars)
641 if not success:
642 raise Exception(f"{log_prefix}Packer validation failed: {stderr}")
644 phase_tracker.mark(
645 build_phase,
646 f"{log_prefix}Building image '{image_name}' (this may take minutes)",
647 )
648 success, output = packer.build("template.pkr.hcl", packer_vars)
649 if not success:
650 raise Exception(f"{log_prefix}Packer build failed: {output}")
652 task_logger.success(
653 f"{log_prefix}Image '{image_name}' built successfully",
654 category=LogCategory.STATUS,
655 )
656 break
657 except Exception as e:
658 raise Exception(f"Packer error: {str(e)}")
659 finally:
660 build_lock.release()
663@celery_app.task(bind=True, name="tasks.deploy_application")
664def deploy_application(
665 self,
666 deployment_id: str,
667 app_id: str,
668 app_git_link: str,
669 release: str,
670 user_vars: dict[str, Any],
671 teams: dict[str, list] = None,
672 openstack_envelope: dict[str, Any] | None = None,
673):
674 """
675 Deploy an application using Terraform and Packer
677 Args:
678 deployment_id: UUID of the deployment
679 app_git_link: Git repo URL
680 release: Tag/Release to checkout
681 user_vars: User variables for Packer/Terraform
682 teams: Teams with user emails {"team_name": [{"email": "user@example.com"}]}
683 openstack_envelope: Encrypted per-user OpenStack credential envelope
684 shipped from the backend. Required for new deploys; the optional
685 default exists only so older queued messages don't crash the
686 worker on rollout (we raise immediately if it's missing).
688 Returns:
689 dict: status, logs, tf_state, commit_info, terraform_outputs
690 """
691 task_logger = get_logger(f"deploy:{deployment_id}", correlation_id=deployment_id)
693 # Wire the per-deployment logger to Celery's event bus. Every buffered
694 # log entry now becomes a ``task-log`` event, and ``task_logger.progress``
695 # emits ``task-progress``. The backend's listener picks both up and
696 # forwards them via the in-process pubsub to any open SSE subscriber.
697 bound_task = self
699 def _emit(event_name: str, payload: dict[str, Any]) -> None:
700 # ``deployment_id`` is duplicated into every event so the backend
701 # listener doesn't need a DB lookup to figure out which deployment
702 # the event belongs to.
703 bound_task.send_event(event_name, deployment_id=deployment_id, **payload)
705 task_logger.set_event_emitter(_emit)
707 # Pessimistic phase set — assumes Packer. Demoted after git clone if
708 # the cloned repo turns out to have no Packer template.
709 phase_tracker = _PhaseTracker(task_logger, _PHASES_WITH_PACKER)
711 repo_path = None
712 tf_state = None
713 outputs = None
714 commit_info = None
715 terraform_dir = None
716 openstack_env: dict[str, str] = {}
717 clouds_config: PerTaskCloudsConfig | None = None
719 # Terraform's pg backend lives in a worker-only Postgres. Configured
720 # at deploy/destroy time by writing a `pg_backend_override.tf` next
721 # to the cloned repo's terraform/ directory. One schema per
722 # deployment isolates state and locks.
723 tfstate_conn_str = settings.TFSTATE_DATABASE_URL or None
724 tfstate_schema = _tfstate_schema_name(deployment_id)
726 # Default teams to empty dict if not provided
727 if teams is None:
728 teams = {}
730 def collect_terraform_state():
731 return collect_terraform_state_helper(
732 terraform_dir, openstack_env, tfstate_conn_str, tfstate_schema, task_logger, local_fallback=True
733 )
735 def collect_terraform_outputs():
736 return collect_terraform_outputs_helper(
737 terraform_dir, openstack_env, tfstate_conn_str, tfstate_schema, task_logger
738 )
740 try:
741 phase_tracker.mark(PHASE_STARTING, "Starting deployment")
742 task_logger.resource_info(
743 "deployment",
744 deployment_id,
745 app_id=app_id,
746 git_url=app_git_link,
747 release=release,
748 user_vars_keys=list(user_vars.keys()),
749 teams_keys=list(teams.keys()),
750 )
752 # Phase 1: OpenStack credentials (envelope only — materialised after
753 # the git clone so the per-task clouds.yaml lives inside repo_path).
754 phase_tracker.mark(PHASE_OPENSTACK_SETUP, "Validating OpenStack credentials")
755 if not openstack_envelope:
756 raise Exception("OpenStack credential envelope missing — user must upload credentials before deploying")
757 task_logger.success(
758 "OpenStack credential envelope received",
759 category=LogCategory.STATUS,
760 )
762 # Phase 2: Git clone
763 phase_tracker.mark(PHASE_GIT_CLONE, "Cloning repository")
764 task_logger.info(f"Cloning repository: {app_git_link}", category=LogCategory.OPERATION)
765 try:
766 repo_path = git_service.clone_release(git_url=app_git_link, deployment_id=deployment_id, tag=release)
768 # Get commit info
769 try:
770 commit_info = _extract_commit_info(repo_path)
771 task_logger.resource_info(
772 "git_commit",
773 commit_info["hash"][:8],
774 hash=commit_info["hash"],
775 message=commit_info["message"],
776 author=commit_info["author"],
777 )
778 task_logger.success(
779 f"Repository cloned at commit {commit_info['hash'][:8]}", category=LogCategory.STATUS
780 )
781 except Exception as e:
782 task_logger.warning(f"Could not extract commit info: {e}", category=LogCategory.WARNING)
784 except Exception as e:
785 raise Exception(f"Git clone failed: {str(e)}")
787 # Materialise the per-task clouds.yaml inside repo_path with mode 0600.
788 # Lives only for the duration of this task; shredded by __exit__.
789 phase_tracker.mark(PHASE_CREDS_MATERIALISE, "Writing per-task clouds.yaml")
790 task_logger.operation_start("openstack_credentials_materialise")
791 clouds_config = PerTaskCloudsConfig(openstack_envelope, work_dir=repo_path)
792 openstack_env = clouds_config.__enter__()
793 task_logger.operation_end("openstack_credentials_materialise", success=True)
794 task_logger.success(
795 "Per-task clouds.yaml written",
796 category=LogCategory.STATUS,
797 )
799 # Cache the built image by commit SHA, not by release tag:
800 # `release` is often a moving ref (e.g. "main"), so the
801 # content-addressed short SHA ensures a new commit misses the cache.
802 # Multi-image apps name each template's image ``<app_id>-<key>-<tag>``;
803 # legacy single-template apps keep the flat ``<app_id>-<tag>`` shape.
804 try:
805 templates = _discover_packer_templates(repo_path)
806 except PackerTemplateDiscoveryError as e:
807 raise Exception(f"Packer template discovery failed: {e}")
809 image_tag = commit_info["hash"][:8] if commit_info and commit_info.get("hash") else release
810 if len(templates) == 1 and templates[0].key == "default":
811 image_names = {"default": f"{app_id}-{image_tag}"}
812 else:
813 image_names = {t.key: f"{app_id}-{t.key}-{image_tag}" for t in templates}
815 # Decide once whether this deployment needs a Packer build, and
816 # adapt the phase total accordingly so the percent bar is honest.
817 # The pessimistic default (assume Packer) was set at task start;
818 # if the cloned repo has no Packer template we drop those three
819 # phases now so the next progress event lands on the right index.
820 # For multi-image apps, ``_phases_for_templates`` expands the
821 # Packer phases per template instead.
822 phase_tracker = _PhaseTracker(task_logger, _phases_for_templates(templates))
824 # The output callback feeds each line of subprocess output into the
825 # task logger as a streaming entry, which then ships it via the
826 # event emitter as a ``task-log`` event. Same callback for Packer
827 # and Terraform — the tool name distinguishes them on the receiver.
828 def _stream_line(tool: str, line: str) -> None:
829 task_logger.tool_output_line(tool, line)
831 # Phase 3: Packer (optional) — guarded by a Redis lock keyed on
832 # (project_id, image_name) so two parallel workers can't both kick
833 # off a build for the same image and end up with duplicate Glance
834 # entries plus wasted compute. For multi-image apps each template
835 # has its own lock + image-exists check, so two workers can build
836 # different images of the same app in parallel.
837 if not templates:
838 task_logger.info("No Packer template found, skipping image build", category=LogCategory.SYSTEM)
839 else:
840 project_id = openstack_envelope.get("project_id") or openstack_envelope.get("project_name") or "default"
841 openstack_service = OpenStackService(env_vars=openstack_env)
842 is_legacy = len(templates) == 1 and templates[0].key == "default"
844 for tmpl in templates:
845 _build_one_packer_image(
846 tmpl,
847 image_name=image_names[tmpl.key],
848 is_legacy=is_legacy,
849 openstack_service=openstack_service,
850 project_id=project_id,
851 repo_path=repo_path,
852 openstack_env=openstack_env,
853 stream_line=_stream_line,
854 user_vars=user_vars,
855 phase_tracker=phase_tracker,
856 task_logger=task_logger,
857 )
859 # Phase 4: Terraform
860 terraform_dir = os.path.join(repo_path, "terraform")
861 if not os.path.exists(terraform_dir):
862 raise Exception(f"Terraform directory not found at {terraform_dir}")
864 terraform = None
865 terraform_vars: dict[str, Any] = {}
866 try:
867 terraform = TerraformExecutor(
868 terraform_dir,
869 env_vars=openstack_env,
870 backend_conn_str=tfstate_conn_str,
871 backend_schema_name=tfstate_schema,
872 output_callback=_stream_line,
873 )
875 phase_tracker.mark(PHASE_TERRAFORM_INIT, "Initializing Terraform")
876 success, stdout, stderr = terraform.init()
877 if not success:
878 # Surface the real reason in the per-deployment log; the
879 # module-level logger only writes to worker stdout, which the
880 # frontend never sees.
881 if stdout:
882 task_logger.command_output("terraform_init_stdout", stdout, returncode=1)
883 if stderr:
884 task_logger.command_output("terraform_init_stderr", stderr, returncode=1)
885 task_logger.error("Terraform init failed", category=LogCategory.ERROR)
886 raise Exception("Terraform init failed")
887 task_logger.success("Terraform initialization completed", category=LogCategory.STATUS)
889 # Merge user_vars with teams for Terraform. Nested structures
890 # pass through encode_terraform_vars unchanged.
891 terraform_vars = {**user_vars["terraform"]} if "terraform" in user_vars else {}
892 # Per-template image-name injection. Legacy single-template
893 # apps see a flat ``image_name``; multi-image apps declare one
894 # ``image_name_<key>`` per template, filled here.
895 _apply_image_name_vars(
896 terraform_vars,
897 image_names,
898 legacy=len(templates) == 1 and templates[0].key == "default",
899 )
900 if teams:
901 terraform_vars["users"] = teams
902 terraform_vars = encode_terraform_vars(terraform_vars)
904 # File-upload variables can balloon a single -var to hundreds
905 # of KB. The Nova metadata service caps cloud-init user_data at
906 # ~64 KB compressed, so warn per-variable above 120 KB to land
907 # the heads-up in the worker log before a boot failure.
908 _log_bytes_per_var_warn = 120 * 1024
909 for _vname, _vstr in terraform_vars.items():
910 if isinstance(_vstr, str) and len(_vstr) > _log_bytes_per_var_warn:
911 task_logger.warning(
912 f"Terraform variable '{_vname}' is "
913 f"{len(_vstr) // 1024} KB encoded — close to the "
914 "cloud-init user_data limit; the VM may fail to "
915 "boot if the template inlines the full value.",
916 category=LogCategory.WARNING,
917 )
919 task_logger.info(
920 "Terraform variable keys",
921 category=LogCategory.OPERATION,
922 keys=list(terraform_vars.keys()),
923 )
925 phase_tracker.mark(PHASE_TERRAFORM_PLAN, "Planning Terraform deployment")
926 success, stdout, stderr = terraform.plan(variables=terraform_vars)
927 if not success:
928 if stdout:
929 task_logger.command_output("terraform_plan_stdout", stdout, returncode=1)
930 if stderr:
931 task_logger.command_output("terraform_plan_stderr", stderr, returncode=1)
932 task_logger.error("Terraform plan failed", category=LogCategory.ERROR)
933 raise Exception("Terraform plan failed")
934 task_logger.success("Terraform plan completed successfully", category=LogCategory.STATUS)
936 phase_tracker.mark(PHASE_TERRAFORM_APPLY, "Applying configuration (this may take minutes)")
937 success, stdout, stderr = terraform.apply(variables=terraform_vars)
938 if not success:
939 if stdout:
940 task_logger.command_output("terraform_apply_stdout", stdout, returncode=1)
941 if stderr:
942 task_logger.command_output("terraform_apply_stderr", stderr, returncode=1)
943 task_logger.error("Terraform apply failed", category=LogCategory.ERROR)
944 raise Exception("Terraform apply failed")
945 task_logger.success("Terraform resources created", category=LogCategory.STATUS)
947 # Collect outputs and state
948 phase_tracker.mark(PHASE_OUTPUTS_AND_CLEANUP, "Collecting outputs")
949 outputs = collect_terraform_outputs()
950 tf_state = collect_terraform_state()
952 if outputs:
953 task_logger.info(
954 "Terraform deployment outputs collected", category=LogCategory.OPERATION, output_count=len(outputs)
955 )
957 except Exception as e:
958 # Try to collect partial results even on failure
959 tf_state = collect_terraform_state()
960 outputs = collect_terraform_outputs()
962 # Best-effort cleanup: a half-finished `terraform apply` typically
963 # leaves orphaned OpenStack resources (networks, ports, volumes)
964 # that quietly eat the project's quota. Run destroy with the same
965 # variables so the apply graph can be reversed; ignore failures
966 # here — we're already in the error path and re-raising below.
967 if terraform is not None and terraform_dir and os.path.exists(terraform_dir):
968 try:
969 task_logger.info(
970 "Running terraform destroy to clean up partially-applied resources",
971 category=LogCategory.OPERATION,
972 )
973 # Rebuild the var-set from the raw user_vars without
974 # the file payloads. Destroy doesn't need the
975 # cloud-init bytes, but Terraform validates every
976 # declared var on every run, so an apply-only file-var
977 # would otherwise reject the cleanup with a schema error.
978 cleanup_tf_vars = _strip_file_vars(user_vars.get("terraform") or {})
979 _apply_image_name_vars(
980 cleanup_tf_vars,
981 image_names,
982 legacy=len(templates) == 1 and templates[0].key == "default",
983 )
984 if teams:
985 cleanup_tf_vars["users"] = teams
986 terraform.destroy(variables=encode_terraform_vars(cleanup_tf_vars))
987 # Refresh state after destroy so the persisted record reflects cleanup.
988 tf_state = collect_terraform_state()
989 except Exception as cleanup_error:
990 task_logger.warning(
991 f"Terraform cleanup failed: {cleanup_error}",
992 category=LogCategory.WARNING,
993 )
995 raise Exception(f"Terraform error: {str(e)}")
997 # The OUTPUTS_AND_CLEANUP progress event was already emitted above;
998 # a second mark would land on the same index, so just log success.
999 task_logger.success(f"Deployment {deployment_id} completed successfully", category=LogCategory.STATUS)
1001 # Log summary
1002 summary = task_logger.get_summary()
1003 task_logger.info("Deployment summary", category=LogCategory.SYSTEM, **summary)
1005 if outputs:
1006 task_logger.info("Terraform deployment output", category=LogCategory.SYSTEM, **outputs)
1008 result = {
1009 "status": "success",
1010 "deployment_id": deployment_id,
1011 "logs": task_logger.get_logs_dict(),
1012 "tf_state": tf_state,
1013 "commit_info": commit_info,
1014 "terraform_outputs": outputs,
1015 }
1017 # Return result (sent via task-succeeded event)
1018 return result
1020 except Exception as e:
1021 task_logger.exception(f"Deployment failed: {str(e)}", exception=e, deployment_id=deployment_id)
1023 # Try to collect any available state/outputs even on failure
1024 if not tf_state:
1025 tf_state = collect_terraform_state()
1026 if not outputs:
1027 outputs = collect_terraform_outputs()
1029 # Raise custom exception with all details
1030 raise Failure(
1031 message=str(e),
1032 deployment_id=deployment_id,
1033 logs_dict=task_logger.get_logs_dict(),
1034 tf_state=tf_state,
1035 commit_info=commit_info,
1036 terraform_outputs=outputs,
1037 )
1039 finally:
1040 # Shred the per-task clouds.yaml first so the credential file is gone
1041 # even if the repository cleanup below fails or hangs.
1042 _cleanup_task_resources(clouds_config, repo_path, task_logger)
1045@celery_app.task(bind=True, name="tasks.destroy_deployment")
1046def destroy_deployment(
1047 self,
1048 deployment_id: str,
1049 app_id: str,
1050 app_git_link: str,
1051 release: str,
1052 user_vars: dict[str, Any],
1053 teams: dict[str, list] = None,
1054 openstack_envelope: dict[str, Any] | None = None,
1055):
1056 """Tear down a deployment via ``terraform destroy``.
1058 Mirrors ``deploy_application``'s setup (git clone at the same release
1059 tag, materialise the per-task clouds.yaml, configure the same pg
1060 backend schema) so Terraform sees the exact same state it built.
1061 Then runs ``terraform destroy -auto-approve`` instead of
1062 ``plan + apply``. Same Packer image is left in Glance so a future
1063 deploy of the same commit doesn't have to rebuild it.
1065 All progress and log events flow through the same ``StructuredLogger``
1066 + Celery custom-event pipeline as deploy, so the frontend's live
1067 SSE stream renders the destroy run identically to a deploy.
1069 Args mirror ``deploy_application`` so the backend can re-dispatch
1070 the same persisted values without translation.
1071 """
1072 task_logger = get_logger(f"destroy:{deployment_id}", correlation_id=deployment_id)
1074 bound_task = self
1076 def _emit(event_name: str, payload: dict[str, Any]) -> None:
1077 bound_task.send_event(event_name, deployment_id=deployment_id, **payload)
1079 task_logger.set_event_emitter(_emit)
1080 phase_tracker = _PhaseTracker(task_logger, _PHASES_DESTROY)
1082 repo_path = None
1083 tf_state: str | None = None
1084 commit_info: dict[str, Any] | None = None
1085 terraform_dir: str | None = None
1086 openstack_env: dict[str, str] = {}
1087 clouds_config: PerTaskCloudsConfig | None = None
1089 tfstate_conn_str = settings.TFSTATE_DATABASE_URL or None
1090 tfstate_schema = _tfstate_schema_name(deployment_id)
1092 if teams is None:
1093 teams = {}
1095 def _stream_line(tool: str, line: str) -> None:
1096 task_logger.tool_output_line(tool, line)
1098 def collect_terraform_state():
1099 return collect_terraform_state_helper(
1100 terraform_dir, openstack_env, tfstate_conn_str, tfstate_schema, task_logger
1101 )
1103 try:
1104 phase_tracker.mark(PHASE_STARTING, "Starting destroy")
1105 task_logger.resource_info(
1106 "deployment",
1107 deployment_id,
1108 app_id=app_id,
1109 git_url=app_git_link,
1110 release=release,
1111 user_vars_keys=list(user_vars.keys()),
1112 teams_keys=list(teams.keys()),
1113 action="destroy",
1114 )
1116 phase_tracker.mark(PHASE_OPENSTACK_SETUP, "Validating OpenStack credentials")
1117 if not openstack_envelope:
1118 raise Exception("OpenStack credential envelope missing — cannot destroy without credentials")
1119 task_logger.success("OpenStack credential envelope received", category=LogCategory.STATUS)
1121 phase_tracker.mark(PHASE_GIT_CLONE, "Cloning repository at original release tag")
1122 task_logger.info(
1123 f"Cloning {app_git_link} at {release} (same ref as the original deploy "
1124 "so terraform code matches the pg-backend state)",
1125 category=LogCategory.OPERATION,
1126 )
1127 try:
1128 repo_path = git_service.clone_release(git_url=app_git_link, deployment_id=deployment_id, tag=release)
1129 try:
1130 commit_info = _extract_commit_info(repo_path)
1131 task_logger.resource_info(
1132 "git_commit",
1133 commit_info["hash"][:8],
1134 hash=commit_info["hash"],
1135 message=commit_info["message"],
1136 author=commit_info["author"],
1137 )
1138 task_logger.success(
1139 f"Repository cloned at commit {commit_info['hash'][:8]}", category=LogCategory.STATUS
1140 )
1141 except Exception as e:
1142 task_logger.warning(f"Could not extract commit info: {e}", category=LogCategory.WARNING)
1143 except Exception as e:
1144 raise Exception(f"Git clone failed: {str(e)}")
1146 phase_tracker.mark(PHASE_CREDS_MATERIALISE, "Writing per-task clouds.yaml")
1147 clouds_config = PerTaskCloudsConfig(openstack_envelope, work_dir=repo_path)
1148 openstack_env = clouds_config.__enter__()
1149 task_logger.success("Per-task clouds.yaml written", category=LogCategory.STATUS)
1151 # Reconstruct the same image_name map the deploy task used so
1152 # the variables match what terraform's state expects to
1153 # validate. Glance still has the image(s), even if we won't be
1154 # using them; the variable just has to be a non-empty string
1155 # that satisfies the HCL declaration. For multi-image apps we
1156 # discover templates here too so the right ``image_name_<key>``
1157 # suffix is injected per template.
1158 try:
1159 templates = _discover_packer_templates(repo_path)
1160 except PackerTemplateDiscoveryError as e:
1161 raise Exception(f"Packer template discovery failed: {e}")
1163 image_tag = commit_info["hash"][:8] if commit_info and commit_info.get("hash") else release
1164 image_names = _build_image_names(templates, app_id, image_tag)
1166 terraform_dir = os.path.join(repo_path, "terraform")
1167 if not os.path.exists(terraform_dir):
1168 raise Exception(f"Terraform directory not found at {terraform_dir}")
1170 # Drop ``@openstack:file:*`` variable values before passing
1171 # the var-set to terraform destroy. Files are only consumed
1172 # at apply-time (cloud-init write_files); destroy doesn't
1173 # need them, but Terraform validates every declared var on
1174 # every run.
1175 terraform_vars = {**user_vars["terraform"]} if "terraform" in user_vars else {}
1176 terraform_vars = _strip_file_vars(terraform_vars)
1177 # Inject the per-template image-name variables. Legacy single
1178 # template (or no Packer at all) keeps the flat ``image_name``;
1179 # multi-template apps get one ``image_name_<key>`` per template.
1180 _apply_image_name_vars(
1181 terraform_vars,
1182 image_names,
1183 legacy=not templates or (len(templates) == 1 and templates[0].key == "default"),
1184 )
1185 if teams:
1186 terraform_vars["users"] = teams
1187 terraform_vars = encode_terraform_vars(terraform_vars)
1189 terraform = TerraformExecutor(
1190 terraform_dir,
1191 env_vars=openstack_env,
1192 backend_conn_str=tfstate_conn_str,
1193 backend_schema_name=tfstate_schema,
1194 output_callback=_stream_line,
1195 )
1197 phase_tracker.mark(PHASE_TERRAFORM_INIT, "Initializing Terraform")
1198 success, stdout, stderr = terraform.init()
1199 if not success:
1200 if stdout:
1201 task_logger.command_output("terraform_init_stdout", stdout, returncode=1)
1202 if stderr:
1203 task_logger.command_output("terraform_init_stderr", stderr, returncode=1)
1204 raise Exception("Terraform init failed")
1205 task_logger.success("Terraform initialization completed", category=LogCategory.STATUS)
1207 phase_tracker.mark(PHASE_TERRAFORM_DESTROY, "Destroying resources")
1208 success, stdout, stderr = terraform.destroy(variables=terraform_vars)
1209 # A data source (e.g. the Glance image lookup) is re-read on every
1210 # destroy refresh. If that image/network was deleted out-of-band,
1211 # the refresh fails with "Your query returned no results" before any
1212 # managed resource is touched. Retry once with -refresh=false so the
1213 # teardown proceeds purely from state. Scoped to this exact error so
1214 # genuine destroy failures still surface.
1215 if not success and "Your query returned no results" in f"{stdout or ''}{stderr or ''}":
1216 task_logger.warning(
1217 "Destroy blocked by a stale data source (image/network deleted "
1218 "out-of-band). Retrying with -refresh=false — resources are torn "
1219 "down from state.",
1220 category=LogCategory.WARNING,
1221 )
1222 success, stdout, stderr = terraform.destroy(variables=terraform_vars, refresh=False)
1223 if not success:
1224 if stdout:
1225 task_logger.command_output("terraform_destroy_stdout", stdout, returncode=1)
1226 if stderr:
1227 task_logger.command_output("terraform_destroy_stderr", stderr, returncode=1)
1228 raise Exception("Terraform destroy failed")
1229 task_logger.success("Terraform resources destroyed", category=LogCategory.STATUS)
1231 phase_tracker.mark(PHASE_CLEANUP, "Pulling final state")
1232 tf_state = collect_terraform_state()
1234 task_logger.success(f"Deployment {deployment_id} destroyed successfully", category=LogCategory.STATUS)
1236 summary = task_logger.get_summary()
1237 task_logger.info("Destroy summary", category=LogCategory.SYSTEM, **summary)
1239 return {
1240 "status": "success",
1241 "deployment_id": deployment_id,
1242 "logs": task_logger.get_logs_dict(),
1243 "tf_state": tf_state,
1244 "commit_info": commit_info,
1245 # No outputs — destroy doesn't produce any. Field is kept for
1246 # event-listener parity with deploy_application's payload.
1247 "terraform_outputs": {},
1248 }
1250 except Exception as e:
1251 task_logger.exception(f"Destroy failed: {str(e)}", exception=e, deployment_id=deployment_id)
1252 if not tf_state:
1253 tf_state = collect_terraform_state()
1254 raise Failure(
1255 message=str(e),
1256 deployment_id=deployment_id,
1257 logs_dict=task_logger.get_logs_dict(),
1258 tf_state=tf_state,
1259 commit_info=commit_info,
1260 terraform_outputs={},
1261 )
1263 finally:
1264 _cleanup_task_resources(clouds_config, repo_path, task_logger)
1267# ----------------------------------------------------------------
1268# PAUSE / RESUME — compute-instance-only lifecycle
1269# ----------------------------------------------------------------
1270#
1271# Both tasks share the destroy preamble (git clone at the same release
1272# tag → per-task clouds.yaml → terraform init pointed at the pg backend)
1273# so we can pull the canonical terraform state and read back which
1274# compute instances belong to this deployment. The hot phase is a
1275# CLI-driven stop/start loop; terraform state is left untouched. Server
1276# discovery goes through the state (not tags) so no app template needs
1277# to opt in, and CLI idempotency lets the loop re-run safely on retry.
1280def _extract_compute_instance_ids(state_json: str | None) -> list[str]:
1281 """Return server IDs from a terraform pg-backend state dump.
1283 Terraform's serialised state shape is
1284 ``{"resources": [{"type": "...", "instances": [{"attributes": {"id": "..."}}]}]}``.
1285 Filtered to ``openstack_compute_instance_v2`` so we only stop/start
1286 Nova servers, not volumes / networks / security groups.
1288 Returns an empty list on any parsing trouble — the caller can then
1289 decide whether "no servers found" is a hard error (deploy never
1290 actually ran) or a no-op success (everything already torn down).
1291 """
1292 if not state_json:
1293 return []
1294 try:
1295 state = json.loads(state_json) if isinstance(state_json, str) else state_json
1296 except (TypeError, json.JSONDecodeError):
1297 return []
1299 ids: list[str] = []
1300 for resource in state.get("resources", []):
1301 if resource.get("type") != "openstack_compute_instance_v2":
1302 continue
1303 for instance in resource.get("instances", []):
1304 attrs = instance.get("attributes") or {}
1305 sid = attrs.get("id")
1306 if sid:
1307 ids.append(sid)
1308 return ids
1311def _run_compute_lifecycle(
1312 self,
1313 deployment_id: str,
1314 app_id: str,
1315 app_git_link: str,
1316 release: str,
1317 user_vars: dict[str, Any],
1318 teams: dict[str, list] | None,
1319 openstack_envelope: dict[str, Any] | None,
1320 *,
1321 action: str, # "pause" | "resume" — only for log/error labels
1322 phases: tuple[str, ...],
1323 server_phase: str,
1324 server_op: str, # "stop" | "start"
1325):
1326 """Shared body for ``pause_deployment`` / ``resume_deployment``.
1328 Mirrors :func:`destroy_deployment`'s preamble exactly so the two
1329 paths stay easy to reason about. Diverges only at the hot phase:
1330 instead of ``terraform destroy``, we pull the state, extract
1331 every compute instance's ID, and shell out to
1332 ``openstack server stop|start`` for each.
1334 Failures during the per-server loop are accumulated and re-raised
1335 once with a list of which servers failed — so the user sees
1336 "stopped 4/5; failed: web-1: locked task" instead of just "pause
1337 failed" without any pointer to which instance is stuck.
1338 """
1339 label = f"{action}:{deployment_id}"
1340 task_logger = get_logger(label, correlation_id=deployment_id)
1342 bound_task = self
1344 def _emit(event_name: str, payload: dict[str, Any]) -> None:
1345 bound_task.send_event(event_name, deployment_id=deployment_id, **payload)
1347 task_logger.set_event_emitter(_emit)
1348 phase_tracker = _PhaseTracker(task_logger, phases)
1350 repo_path = None
1351 terraform_dir: str | None = None
1352 openstack_env: dict[str, str] = {}
1353 clouds_config: PerTaskCloudsConfig | None = None
1355 tfstate_conn_str = settings.TFSTATE_DATABASE_URL or None
1356 tfstate_schema = _tfstate_schema_name(deployment_id)
1358 if teams is None:
1359 teams = {}
1361 def _stream_line(tool: str, line: str) -> None:
1362 task_logger.tool_output_line(tool, line)
1364 try:
1365 phase_tracker.mark(PHASE_STARTING, f"Starting {action}")
1366 task_logger.resource_info(
1367 "deployment",
1368 deployment_id,
1369 app_id=app_id,
1370 git_url=app_git_link,
1371 release=release,
1372 action=action,
1373 )
1375 phase_tracker.mark(PHASE_OPENSTACK_SETUP, "Validating OpenStack credentials")
1376 if not openstack_envelope:
1377 raise Exception(f"OpenStack credential envelope missing — cannot {action} without credentials")
1378 task_logger.success("OpenStack credential envelope received", category=LogCategory.STATUS)
1380 phase_tracker.mark(PHASE_GIT_CLONE, "Cloning repository at original release tag")
1381 try:
1382 repo_path = git_service.clone_release(
1383 git_url=app_git_link,
1384 deployment_id=deployment_id,
1385 tag=release,
1386 )
1387 task_logger.success("Repository cloned", category=LogCategory.STATUS)
1388 except Exception as e:
1389 raise Exception(f"Git clone failed: {str(e)}")
1391 phase_tracker.mark(PHASE_CREDS_MATERIALISE, "Writing per-task clouds.yaml")
1392 clouds_config = PerTaskCloudsConfig(openstack_envelope, work_dir=repo_path)
1393 openstack_env = clouds_config.__enter__()
1394 task_logger.success("Per-task clouds.yaml written", category=LogCategory.STATUS)
1396 terraform_dir = os.path.join(repo_path, "terraform")
1397 if not os.path.exists(terraform_dir):
1398 raise Exception(f"Terraform directory not found at {terraform_dir}")
1400 terraform = TerraformExecutor(
1401 terraform_dir,
1402 env_vars=openstack_env,
1403 backend_conn_str=tfstate_conn_str,
1404 backend_schema_name=tfstate_schema,
1405 output_callback=_stream_line,
1406 )
1408 phase_tracker.mark(PHASE_TERRAFORM_INIT, "Initializing Terraform")
1409 success, stdout, stderr = terraform.init()
1410 if not success:
1411 if stdout:
1412 task_logger.command_output("terraform_init_stdout", stdout, returncode=1)
1413 if stderr:
1414 task_logger.command_output("terraform_init_stderr", stderr, returncode=1)
1415 raise Exception("Terraform init failed")
1416 task_logger.success("Terraform initialization completed", category=LogCategory.STATUS)
1418 # Pull the canonical state from the pg backend, then walk it
1419 # to find every compute instance attached to this deployment.
1420 # An empty list usually means the deployment never reached a
1421 # successful apply — surface that as a hard error rather than
1422 # a silent no-op so the user doesn't think pause "worked" on
1423 # an empty deployment.
1424 state_dump = terraform.state_pull()
1425 server_ids = _extract_compute_instance_ids(state_dump)
1426 if not server_ids:
1427 raise Exception(
1428 "No compute instances found in terraform state — "
1429 f"nothing to {action}. The deployment may have been "
1430 "torn down already or never reached a successful apply."
1431 )
1432 task_logger.info(
1433 f"{len(server_ids)} compute instance(s) found",
1434 category=LogCategory.OPERATION,
1435 server_ids=server_ids,
1436 )
1438 phase_tracker.mark(
1439 server_phase,
1440 f"{'Stopping' if server_op == 'stop' else 'Starting'} {len(server_ids)} server(s)",
1441 )
1443 openstack_service = OpenStackService(env_vars=openstack_env)
1444 op_method = openstack_service.server_stop if server_op == "stop" else openstack_service.server_start
1446 failures: list[tuple[str, str]] = []
1447 for sid in server_ids:
1448 # Optional pre-flight: log the human name + power state so
1449 # the per-deployment log is readable. We never fail on
1450 # show() — it's purely cosmetic.
1451 info = openstack_service.server_show(sid)
1452 label_str = f"{info.get('name', sid)}" if info else sid
1453 current = info.get("status") if info else None
1454 task_logger.info(
1455 f"{server_op} {label_str} (status: {current or 'unknown'})",
1456 category=LogCategory.OPERATION,
1457 server_id=sid,
1458 )
1460 ok, err = op_method(sid)
1461 if ok:
1462 task_logger.success(
1463 f"{label_str}: {server_op} OK",
1464 category=LogCategory.STATUS,
1465 )
1466 else:
1467 task_logger.error(
1468 f"{label_str}: {server_op} failed: {err}",
1469 category=LogCategory.ERROR,
1470 server_id=sid,
1471 )
1472 failures.append((sid, err or "unknown error"))
1474 if failures:
1475 joined = "; ".join(f"{sid}: {err}" for sid, err in failures)
1476 raise Exception(f"{action} failed for {len(failures)}/{len(server_ids)} server(s): {joined}")
1478 phase_tracker.mark(PHASE_CLEANUP, "Pulling final state snapshot")
1479 # State doesn't change for pause/resume (the resources still
1480 # exist, just in a different power state), but we pull it
1481 # again so the task row gets a fresh snapshot for debugging.
1482 try:
1483 tf_state_post = terraform.state_pull()
1484 except Exception as e:
1485 task_logger.warning(
1486 f"Could not pull terraform state post-{action}: {e}",
1487 category=LogCategory.WARNING,
1488 )
1489 tf_state_post = state_dump
1491 task_logger.success(
1492 f"Deployment {deployment_id} {action}d successfully",
1493 category=LogCategory.STATUS,
1494 )
1496 return {
1497 "status": "success",
1498 "deployment_id": deployment_id,
1499 "logs": task_logger.get_logs_dict(),
1500 "tf_state": tf_state_post,
1501 "commit_info": None,
1502 # Pause/resume don't generate or change terraform outputs —
1503 # field is kept for event-listener parity with the deploy
1504 # / destroy payload shape.
1505 "terraform_outputs": {},
1506 }
1508 except Exception as e:
1509 task_logger.exception(f"{action} failed: {str(e)}", exception=e, deployment_id=deployment_id)
1510 raise Failure(
1511 message=str(e),
1512 deployment_id=deployment_id,
1513 logs_dict=task_logger.get_logs_dict(),
1514 tf_state=None,
1515 commit_info=None,
1516 terraform_outputs={},
1517 )
1519 finally:
1520 _cleanup_task_resources(clouds_config, repo_path, task_logger)
1523@celery_app.task(bind=True, name="tasks.pause_deployment")
1524def pause_deployment(
1525 self,
1526 deployment_id: str,
1527 app_id: str,
1528 app_git_link: str,
1529 release: str,
1530 user_vars: dict[str, Any],
1531 teams: dict[str, list] = None,
1532 openstack_envelope: dict[str, Any] | None = None,
1533):
1534 """Halt a deployment by stopping all of its compute instances.
1536 Volumes and networks are untouched, so resume restores the same
1537 instances byte-for-byte. The terraform state is also untouched,
1538 so a subsequent destroy proceeds normally (terraform destroy is
1539 happy to tear down SHUTOFF instances).
1540 """
1541 return _run_compute_lifecycle(
1542 self,
1543 deployment_id,
1544 app_id,
1545 app_git_link,
1546 release,
1547 user_vars,
1548 teams,
1549 openstack_envelope,
1550 action="pause",
1551 phases=_PHASES_PAUSE,
1552 server_phase=PHASE_SERVER_STOP,
1553 server_op="stop",
1554 )
1557@celery_app.task(bind=True, name="tasks.resume_deployment")
1558def resume_deployment(
1559 self,
1560 deployment_id: str,
1561 app_id: str,
1562 app_git_link: str,
1563 release: str,
1564 user_vars: dict[str, Any],
1565 teams: dict[str, list] = None,
1566 openstack_envelope: dict[str, Any] | None = None,
1567):
1568 """Resume a paused deployment by starting all of its compute instances.
1570 Mirrors :func:`pause_deployment`'s preamble exactly so the two
1571 code paths stay symmetric and easy to compare side-by-side.
1572 """
1573 return _run_compute_lifecycle(
1574 self,
1575 deployment_id,
1576 app_id,
1577 app_git_link,
1578 release,
1579 user_vars,
1580 teams,
1581 openstack_envelope,
1582 action="resume",
1583 phases=_PHASES_RESUME,
1584 server_phase=PHASE_SERVER_START,
1585 server_op="start",
1586 )
1589# ----------------------------------------------------------------
1590# REDEPLOY ONE RESOURCE
1591# ----------------------------------------------------------------
1592#
1593# Replace exactly one compute instance via
1594# ``terraform apply -replace=<addr> -target=<addr>``. Everything else in
1595# the deployment stays untouched. The backend whitelists the address
1596# against the cached TF state before dispatch; we re-check the address
1597# shape here as defense in depth. Same per-task clouds.yaml + pg backend
1598# schema as deploy/destroy, so the apply sees the same state file.
1600_REDEPLOY_ADDRESS_RE = re.compile(
1601 r"""^
1602 [A-Za-z_][A-Za-z0-9_]*
1603 \.[A-Za-z_][A-Za-z0-9_-]*
1604 (?:\[(?:\d+|"[^"\\]+")\])?
1605 $""",
1606 re.VERBOSE,
1607)
1610def _build_current_roster(teams: dict[str, list]) -> tuple[set[str], set[str]]:
1611 """Compute the legal slot-key sets for the current roster.
1613 Returns a ``(team_keys, user_keys)`` tuple:
1615 * ``team_keys`` — every team name currently present. These are the
1616 valid slot keys for ``var_scope=team``.
1617 * ``user_keys`` — composite ``"<team>-<email>"`` keys for every
1618 user currently rostered to a team. These are the valid slot keys
1619 for ``var_scope=user``.
1621 Roster entries can either be plain strings (email addresses) or
1622 dicts with an ``email`` key — matches the shape the backend ships
1623 in ``teams`` (see ``_attach_files_to_user_input``). Anything else is
1624 skipped defensively.
1625 """
1626 team_keys: set[str] = set()
1627 user_keys: set[str] = set()
1628 for team_name, members in (teams or {}).items():
1629 if not team_name:
1630 continue
1631 team_keys.add(team_name)
1632 if not isinstance(members, list):
1633 continue
1634 for member in members:
1635 email = member if isinstance(member, str) else (member.get("email") if isinstance(member, dict) else None)
1636 if not email:
1637 continue
1638 user_keys.add(f"{team_name}-{email}")
1639 return team_keys, user_keys
1642def _reconcile_scoped_vars_to_roster(
1643 terraform_vars: dict[str, Any],
1644 teams: dict[str, list],
1645 task_logger: Any,
1646) -> dict[str, Any]:
1647 """Drop scoped-map entries whose slot keys no longer match the roster.
1649 Redeploy replays the originally-persisted ``user_vars["terraform"]``
1650 blob, but the team/user roster may have shifted since the initial
1651 deploy (members added or removed, teams renamed). A scoped variable
1652 keyed on the old roster would then ship Terraform a map containing
1653 orphan keys — at best a noisy diff, at worst a type/required
1654 failure that blocks the replace.
1656 Heuristic: a value is considered scoped when it's a non-empty
1657 ``dict`` whose keys form a subset of either the team-name roster
1658 (``var_scope=team``) or the ``<team>-<user>`` composite roster
1659 (``var_scope=user``). On match we intersect the value's keys with
1660 the current roster and drop the orphans. Maps that don't match the
1661 heuristic — e.g. file-shape vars or the ``users`` injection — are
1662 left untouched. Every drop is announced in the task log so the
1663 operator sees which slots were retired.
1665 A value that becomes empty after intersection is dropped from the
1666 var-set entirely; Terraform validation handles the
1667 missing-required case from there (it can pick up a declared
1668 default or surface the required-but-missing error properly).
1669 """
1670 if not terraform_vars:
1671 return terraform_vars
1673 team_keys, user_keys = _build_current_roster(teams)
1674 if not team_keys and not user_keys:
1675 # Nothing rostered — can't reconcile, leave the var-set alone.
1676 return terraform_vars
1678 reconciled: dict[str, Any] = {}
1679 for name, value in terraform_vars.items():
1680 # Only dict-shaped, non-empty values can be scoped maps. Skip
1681 # the ``users`` injection — we set that ourselves from ``teams``
1682 # right after this and it's not an app-defined scoped var.
1683 if name == "users" or not isinstance(value, dict) or not value:
1684 reconciled[name] = value
1685 continue
1686 # File-shape values (see ``_looks_like_file_var_value``) are
1687 # already keyed by slot but use a different content contract;
1688 # let the regular file-strip handle them.
1689 if _looks_like_file_var_value(value):
1690 reconciled[name] = value
1691 continue
1693 slot_keys = set(value.keys())
1694 # Pick the roster axis whose universe best matches the slot
1695 # keys. Subset wins outright; otherwise pick the axis with the
1696 # larger overlap so a partially-stale map still gets cleaned.
1697 team_overlap = slot_keys & team_keys
1698 user_overlap = slot_keys & user_keys
1699 if slot_keys <= team_keys and team_keys:
1700 allowed = team_keys
1701 elif slot_keys <= user_keys and user_keys or len(user_overlap) >= len(team_overlap) and user_overlap:
1702 allowed = user_keys
1703 elif team_overlap:
1704 allowed = team_keys
1705 else:
1706 # No overlap with either roster axis — leave the value
1707 # alone. Probably a non-scoped map(string,...) variable
1708 # the user explicitly populated.
1709 reconciled[name] = value
1710 continue
1712 kept = {k: v for k, v in value.items() if k in allowed}
1713 dropped = sorted(slot_keys - allowed)
1714 if dropped:
1715 task_logger.warning(
1716 f"Redeploy roster reconciliation: dropped {len(dropped)} "
1717 f"orphan slot(s) from variable '{name}': {dropped}",
1718 category=LogCategory.WARNING,
1719 variable=name,
1720 dropped_slots=dropped,
1721 )
1722 if kept:
1723 reconciled[name] = kept
1724 else:
1725 # All slots orphaned — drop the var entirely so terraform
1726 # validation can fall back to the declared default (if any)
1727 # or surface a proper required-but-missing error.
1728 task_logger.warning(
1729 f"Redeploy roster reconciliation: variable '{name}' has "
1730 "no surviving slots after roster intersection — falling "
1731 "back to its declared default (or required-but-missing).",
1732 category=LogCategory.WARNING,
1733 variable=name,
1734 )
1735 return reconciled
1738@celery_app.task(bind=True, name="tasks.redeploy_resource")
1739def redeploy_resource(
1740 self,
1741 deployment_id: str,
1742 app_id: str,
1743 app_git_link: str,
1744 release: str,
1745 user_vars: dict[str, Any],
1746 teams: dict[str, list] = None,
1747 openstack_envelope: dict[str, Any] | None = None,
1748 resource_address: str | None = None,
1749):
1750 """Replace ONE compute instance via ``-target`` + ``-replace``.
1752 Args mirror ``deploy_application`` so the backend's
1753 ``_dispatch_lifecycle_task`` can ship the same persisted state.
1754 The extra ``resource_address`` carries the Terraform state address
1755 (e.g. ``openstack_compute_instance_v2.team_ide["Team-A"]``) the
1756 user clicked.
1758 Returns the same payload shape as deploy/destroy so the celery
1759 event listener stays generic.
1760 """
1761 task_logger = get_logger(f"redeploy:{deployment_id}", correlation_id=deployment_id)
1763 bound_task = self
1765 def _emit(event_name: str, payload: dict[str, Any]) -> None:
1766 bound_task.send_event(event_name, deployment_id=deployment_id, **payload)
1768 task_logger.set_event_emitter(_emit)
1769 phase_tracker = _PhaseTracker(task_logger, _PHASES_REDEPLOY)
1771 repo_path: str | None = None
1772 tf_state: str | None = None
1773 outputs: dict[str, Any] | None = None
1774 commit_info: dict[str, Any] | None = None
1775 terraform_dir: str | None = None
1776 openstack_env: dict[str, str] = {}
1777 clouds_config: PerTaskCloudsConfig | None = None
1779 tfstate_conn_str = settings.TFSTATE_DATABASE_URL or None
1780 tfstate_schema = _tfstate_schema_name(deployment_id)
1782 if teams is None:
1783 teams = {}
1785 def _stream_line(tool: str, line: str) -> None:
1786 task_logger.tool_output_line(tool, line)
1788 def collect_terraform_state():
1789 return collect_terraform_state_helper(
1790 terraform_dir, openstack_env, tfstate_conn_str, tfstate_schema, task_logger
1791 )
1793 def collect_terraform_outputs():
1794 return collect_terraform_outputs_helper(
1795 terraform_dir, openstack_env, tfstate_conn_str, tfstate_schema, task_logger
1796 )
1798 try:
1799 # Validate the address shape before we do any work. Backend
1800 # already whitelisted the address against the cached state, but
1801 # we double-check the shape so an empty / malformed string from
1802 # a misconfigured caller doesn't reach the CLI.
1803 if not resource_address or not _REDEPLOY_ADDRESS_RE.match(resource_address):
1804 raise Exception(f"redeploy_resource called with invalid resource_address: " f"{resource_address!r}")
1806 phase_tracker.mark(PHASE_STARTING, f"Starting redeploy of {resource_address}")
1807 task_logger.resource_info(
1808 "deployment",
1809 deployment_id,
1810 app_id=app_id,
1811 git_url=app_git_link,
1812 release=release,
1813 user_vars_keys=list(user_vars.keys()),
1814 teams_keys=list(teams.keys()),
1815 action="redeploy",
1816 resource_address=resource_address,
1817 )
1819 phase_tracker.mark(PHASE_OPENSTACK_SETUP, "Validating OpenStack credentials")
1820 if not openstack_envelope:
1821 raise Exception("OpenStack credential envelope missing — cannot redeploy without credentials")
1822 task_logger.success("OpenStack credential envelope received", category=LogCategory.STATUS)
1824 phase_tracker.mark(PHASE_GIT_CLONE, "Cloning repository at original release tag")
1825 task_logger.info(
1826 f"Cloning {app_git_link} at {release} (same ref as the original deploy "
1827 "so terraform code matches the pg-backend state)",
1828 category=LogCategory.OPERATION,
1829 )
1830 try:
1831 repo_path = git_service.clone_release(git_url=app_git_link, deployment_id=deployment_id, tag=release)
1832 try:
1833 commit_info = _extract_commit_info(repo_path)
1834 task_logger.success(
1835 f"Repository cloned at commit {commit_info['hash'][:8]}",
1836 category=LogCategory.STATUS,
1837 )
1838 except Exception as e:
1839 task_logger.warning(f"Could not extract commit info: {e}", category=LogCategory.WARNING)
1840 except Exception as e:
1841 raise Exception(f"Git clone failed: {str(e)}")
1843 phase_tracker.mark(PHASE_CREDS_MATERIALISE, "Writing per-task clouds.yaml")
1844 clouds_config = PerTaskCloudsConfig(openstack_envelope, work_dir=repo_path)
1845 openstack_env = clouds_config.__enter__()
1846 task_logger.success("Per-task clouds.yaml written", category=LogCategory.STATUS)
1848 # Reconstruct the same image_name map the original deploy
1849 # used so the apply's variable validation matches.
1850 # ``image_name`` (or ``image_name_<key>`` per template for
1851 # multi-image apps) is a HCL contract variable; a mismatch
1852 # would surface as a noisy "var changed" diff that wouldn't
1853 # actually apply anything.
1854 try:
1855 templates = _discover_packer_templates(repo_path)
1856 except PackerTemplateDiscoveryError as e:
1857 raise Exception(f"Packer template discovery failed: {e}")
1859 image_tag = commit_info["hash"][:8] if commit_info and commit_info.get("hash") else release
1860 image_names = _build_image_names(templates, app_id, image_tag)
1862 terraform_dir = os.path.join(repo_path, "terraform")
1863 if not os.path.exists(terraform_dir):
1864 raise Exception(f"Terraform directory not found at {terraform_dir}")
1866 # Build the terraform var-set like the original deploy. We KEEP
1867 # file variables here: ``terraform apply -replace`` recreates the
1868 # targeted VM, so cloud-init runs fresh and needs the original
1869 # ``write_files`` payload. The backend pre-filters these vars for
1870 # non-recreating lifecycles (destroy/pause/resume) — see
1871 # ``_dispatch_lifecycle_task`` in backend/app/routers/deployments.py.
1872 terraform_vars = {**user_vars["terraform"]} if "terraform" in user_vars else {}
1873 # The persisted ``user_vars`` were keyed on the roster at deploy
1874 # time. Membership may have shifted since (team renames, members
1875 # added/removed); ship the apply only the slots that still match
1876 # the current roster so terraform doesn't choke on orphan keys.
1877 terraform_vars = _reconcile_scoped_vars_to_roster(terraform_vars, teams, task_logger)
1878 # Inject the per-template image-name variables (legacy: flat
1879 # ``image_name``; multi: one ``image_name_<key>`` per template).
1880 _apply_image_name_vars(
1881 terraform_vars,
1882 image_names,
1883 legacy=not templates or (len(templates) == 1 and templates[0].key == "default"),
1884 )
1885 if teams:
1886 terraform_vars["users"] = teams
1887 terraform_vars = encode_terraform_vars(terraform_vars)
1889 terraform = TerraformExecutor(
1890 terraform_dir,
1891 env_vars=openstack_env,
1892 backend_conn_str=tfstate_conn_str,
1893 backend_schema_name=tfstate_schema,
1894 output_callback=_stream_line,
1895 )
1897 phase_tracker.mark(PHASE_TERRAFORM_INIT, "Initializing Terraform")
1898 success, stdout, stderr = terraform.init()
1899 if not success:
1900 if stdout:
1901 task_logger.command_output("terraform_init_stdout", stdout, returncode=1)
1902 if stderr:
1903 task_logger.command_output("terraform_init_stderr", stderr, returncode=1)
1904 raise Exception("Terraform init failed")
1905 task_logger.success("Terraform initialization completed", category=LogCategory.STATUS)
1907 phase_tracker.mark(
1908 PHASE_TERRAFORM_APPLY,
1909 f"Applying replace for {resource_address}",
1910 )
1911 # ``-replace`` taints the single resource so terraform plans a
1912 # destroy+create on it; ``-target`` scopes the apply to that
1913 # resource (and its dependencies) so the rest of the graph is
1914 # untouched.
1915 success, stdout, stderr = terraform.apply(
1916 variables=terraform_vars,
1917 targets=[resource_address],
1918 replace=[resource_address],
1919 )
1920 if not success:
1921 if stdout:
1922 task_logger.command_output("terraform_apply_stdout", stdout, returncode=1)
1923 if stderr:
1924 task_logger.command_output("terraform_apply_stderr", stderr, returncode=1)
1925 raise Exception("Terraform apply (replace) failed")
1926 task_logger.success(
1927 f"Resource {resource_address} replaced",
1928 category=LogCategory.STATUS,
1929 )
1931 phase_tracker.mark(PHASE_CLEANUP, "Pulling final state")
1932 tf_state = collect_terraform_state()
1933 outputs = collect_terraform_outputs()
1935 task_logger.success(
1936 f"Deployment {deployment_id} redeploy of {resource_address} completed",
1937 category=LogCategory.STATUS,
1938 )
1940 return {
1941 "status": "success",
1942 "deployment_id": deployment_id,
1943 "logs": task_logger.get_logs_dict(),
1944 "tf_state": tf_state,
1945 "commit_info": commit_info,
1946 "terraform_outputs": outputs or {},
1947 }
1949 except Exception as e:
1950 task_logger.exception(f"Redeploy failed: {str(e)}", exception=e, deployment_id=deployment_id)
1951 if not tf_state:
1952 tf_state = collect_terraform_state()
1953 raise Failure(
1954 message=str(e),
1955 deployment_id=deployment_id,
1956 logs_dict=task_logger.get_logs_dict(),
1957 tf_state=tf_state,
1958 commit_info=commit_info,
1959 terraform_outputs=outputs or {},
1960 )
1962 finally:
1963 _cleanup_task_resources(clouds_config, repo_path, task_logger)