Coverage for app/routers/deployments.py: 71.13%
523 statements
« prev ^ index » next coverage.py v7.14.1, created at 2026-07-25 15:51 +0000
1import asyncio
2import base64
3import binascii
4import json
5import logging
6import re
7from collections.abc import AsyncIterator
8from dataclasses import asdict
9from uuid import UUID
11from fastapi import APIRouter, Depends, HTTPException, Request, Response, status
12from fastapi.responses import JSONResponse, StreamingResponse
13from sqlalchemy import desc
14from sqlalchemy.orm import Session
16from app.crud import apps as crud_apps
17from app.crud import deployments as crud_deployments
18from app.crud import locks as crud_locks
19from app.crud import openstack_credentials as crud_openstack_credentials
20from app.crud import teams as crud_teams
21from app.crud import users as crud_users
22from app.database import get_db
23from app.models import Deployment, TaskStatus, TaskType, User, UserRole
24from app.models import Task as TaskModel # for ad-hoc state queries
25from app.schemas import (
26 DeploymentCreate,
27 DeploymentDetail,
28 DeploymentOutputs,
29 DeploymentResourceListResponse,
30 DeploymentResourceSchema,
31 DeploymentResponse,
32 DeploymentTeamMember,
33 DeploymentTeamResponse,
34 MyAccessResponse,
35 TaskSummary,
36)
37from app.services import deployment_notifier, email_service
38from app.services import lifecycle as lifecycle_service
39from app.services import task_service as task_service_module
40from app.services.deployment_pubsub import pubsub
41from app.services.deployment_status import (
42 build_resource_detail,
43 build_resource_views,
44)
45from app.services.tf_state_parser import parse_tf_state
46from app.utils.capabilities import (
47 can_view_deployment_owner,
48 ensure_operate_deployment,
49 ensure_resend_access,
50 ensure_view_app,
51 ensure_view_deployment_owner,
52 get_my_course_teacher_ids,
53)
54from app.utils.keycloak_auth import get_current_user_keycloak
55from app.utils.permissions import (
56 ensure_deployment_access,
57 is_deployment_owner_view,
58)
60logger = logging.getLogger(__name__)
61router = APIRouter()
64def _list_course_scope_deployments(
65 db: Session,
66 current_user: User,
67 skip: int,
68 limit: int,
69 app_id: UUID | None,
70 status_filter: str | None,
71 student: UUID | None,
72) -> list:
73 """Resolve the ``?scope=course`` deployment listing for staff callers.
75 Lists every deployment whose owner sits in one of the requestor's
76 taught courses. Admins use the same code path for symmetry — usually
77 their set is empty. Returns the raw ``Deployment`` rows; the caller
78 runs the shared enrichment loop over the result.
79 """
80 my_courses = get_my_course_teacher_ids(current_user, db)
81 if current_user.role == UserRole.ADMIN:
82 # Admins always see everything inside the chosen scope.
83 # We still need the course-id filter to make ``?scope=course``
84 # narrow the listing in some way — otherwise the param is a
85 # no-op for admins. The implementation: pull every course
86 # the admin is registered as course-teacher for (typically
87 # empty), and fall back to the union with the explicit
88 # ``student`` filter so the route still does something
89 # useful in the common case of an admin requesting a
90 # specific student's deployments via the profile page.
91 pass
92 if not my_courses and current_user.role != UserRole.ADMIN:
93 # Teacher with no course-teacher rows — the course scope
94 # is empty by definition. Return an empty page rather
95 # than the teacher's own owned set, because that would
96 # mask the absence of any teacher-course assignment.
97 return []
99 # Resolve the set of candidate student userIds: every user
100 # whose ``courseId`` falls inside ``my_courses``. When the
101 # caller also passed ``?student=<id>``, narrow to that single
102 # user IFF they actually sit inside one of those courses.
103 student_q = db.query(User.userId).filter(
104 User.courseId.in_(my_courses)
105 )
106 if student is not None:
107 # ``?student=<id>``: require the student to be inside one
108 # of the teacher's courses; otherwise we'd leak that
109 # ``student`` exists at all to a teacher who can't see them.
110 student_q = student_q.filter(User.userId == student)
111 owner_ids = [row[0] for row in student_q.all()]
113 if current_user.role == UserRole.ADMIN and not owner_ids:
114 # Admin path with empty course set + no student filter →
115 # show the admin's own owned set instead of an empty page.
116 if student is None:
117 return crud_deployments.get_deployments(
118 db,
119 skip=skip,
120 limit=limit,
121 user_id=current_user.userId,
122 app_id=app_id,
123 status=status_filter,
124 )
125 return []
126 if not owner_ids:
127 # Teacher in scope mode but their courses are empty, or
128 # the named ``student`` isn't in any of their courses —
129 # empty page, no leak about that student's existence.
130 return []
132 # We can't easily widen the existing get_deployments
133 # signature to accept an owner_id IN clause without
134 # disturbing the other branches, so we build the query
135 # inline here. Same soft-delete + app_id + status
136 # semantics as the helper.
137 q = db.query(Deployment).filter(Deployment.deleted_at.is_(None))
138 q = q.filter(Deployment.userId.in_(owner_ids))
139 if app_id:
140 q = q.filter(Deployment.appId == app_id)
141 # Reuse the helper's status-filter implementation by
142 # forwarding to ``get_deployments`` with a synthetic
143 # ``user_id`` of None and a post-filter — but the helper
144 # short-circuits on user_id, so simplest is to inline the
145 # ordering/pagination and skip the status filter here. A
146 # course-teacher list view rarely needs status filtering
147 # in the index; the per-deployment detail page handles
148 # status-specific UX.
149 q = q.order_by(desc(Deployment.deploymentId))
150 return q.offset(skip).limit(limit).all()
153# ----------------------------------------------------------------
154# GET ALL DEPLOYMENTS
155# ----------------------------------------------------------------
156@router.get("/", response_model=list[DeploymentResponse])
157def list_deployments(
158 skip: int = 0,
159 limit: int = 100,
160 app_id: UUID | None = None,
161 status_filter: str | None = None,
162 scope: str | None = None,
163 student: UUID | None = None,
164 db: Session = Depends(get_db),
165 current_user: User = Depends(get_current_user_keycloak)
166):
167 """List deployments visible to the caller.
169 Visibility rules:
170 * **Admins**: their own owned set (default), or — with
171 ``?scope=course`` — every deployment whose owner sits in a
172 course the admin is a designated teacher of (rare; admins are
173 usually not in ``course_teachers`` rows, but the path is
174 symmetric with the teacher one for consistency).
175 * **Teachers** (default ``scope`` omitted): deployments they
176 created (their own owned set). Cross-user listing is
177 intentional UX: a teacher opens an individual deployment via
178 direct link or via a student's profile page, not from this
179 index.
180 * **Teachers** (``?scope=course``): every non-deleted deployment
181 whose owner sits in a course they teach (course-teacher
182 inspect right). Optional ``?student=<userId>`` narrows the
183 listing to one course-member's deployments,
184 which is what the student-profile page uses to render the
185 deployments list of a single student under the teacher's care.
186 * **Students** (and any non-staff role): deployments they own
187 OR are a member of — either via a team mapping or a direct
188 ``UserToDeployment`` row.
190 The ``scope`` parameter is accepted only for teachers and admins;
191 students passing it get the standard student listing back (the
192 parameter is ignored, not rejected, to keep the API forgiving
193 against query-string-builder bugs in the frontend).
194 """
195 is_staff = current_user.role in (UserRole.TEACHER, UserRole.ADMIN)
196 use_course_scope = is_staff and scope == "course"
198 if use_course_scope:
199 deployments = _list_course_scope_deployments(
200 db,
201 current_user,
202 skip=skip,
203 limit=limit,
204 app_id=app_id,
205 status_filter=status_filter,
206 student=student,
207 )
208 elif is_staff:
209 deployments = crud_deployments.get_deployments(
210 db,
211 skip=skip,
212 limit=limit,
213 user_id=current_user.userId,
214 app_id=app_id,
215 status=status_filter,
216 )
217 else:
218 deployments = crud_deployments.get_deployments(
219 db,
220 skip=skip,
221 limit=limit,
222 member_user_id=current_user.userId,
223 app_id=app_id,
224 status=status_filter,
225 )
227 # Enrich with status and created_at from tasks. The summary is
228 # bulk-fetched in two queries (latest + first task per deployment via
229 # window functions) so the list endpoint stays at a constant query
230 # count regardless of page size.
231 task_summary = crud_deployments.bulk_get_task_summary(
232 db, [d.deploymentId for d in deployments]
233 )
235 result = []
236 for deployment in deployments:
237 # Pull the latest-task ``(status, type)`` and the first-task
238 # timestamp out of the bulk map. Deployments without any task
239 # yet (new row, dispatch in flight) map to ``(None, None, None)``
240 # — ``derive_status`` returns None for that, which the schema
241 # accepts (``status: str | None``).
242 latest_status, latest_type, first_created_at = task_summary.get(
243 deployment.deploymentId, (None, None, None)
244 )
245 status_value = crud_deployments.derive_status(latest_status, latest_type)
247 # Parse userInputVar JSON string back to dict if it exists.
248 # File uploads are stripped down to metadata here so the list
249 # view doesn't ship megabytes of base64 to the browser; the
250 # detail endpoint follows the same rule, and the dedicated
251 # download route is the only path that returns raw bytes.
252 user_input_var_parsed = _parse_and_strip_user_input(deployment.userInputVar)
254 result.append(DeploymentResponse(
255 deploymentId=deployment.deploymentId,
256 name=deployment.name,
257 appId=deployment.appId,
258 userId=deployment.userId,
259 releaseTag=deployment.releaseTag,
260 userInputVar=user_input_var_parsed,
261 status=status_value,
262 created_at=first_created_at,
263 ))
265 return result
268# ----------------------------------------------------------------
269# GET DEPLOYMENT BY ID (Full Details)
270# ----------------------------------------------------------------
271@router.get("/{deployment_id}", response_model=DeploymentDetail)
272def get_deployment(
273 deployment_id: UUID,
274 include_logs: bool = False,
275 db: Session = Depends(get_db),
276 current_user: User = Depends(get_current_user_keycloak)
277):
278 """
279 Get deployment by ID with full details including:
280 - User and App relations
281 - Teams with members
282 - Latest task status
283 - Terraform outputs
284 - Optionally: full logs (use include_logs=true)
285 """
286 deployment = crud_deployments.get_deployment_with_details(db, deployment_id)
287 if not deployment:
288 raise HTTPException(
289 status_code=status.HTTP_404_NOT_FOUND,
290 detail="Deployment not found"
291 )
293 # Check access permission
294 ensure_deployment_access(deployment, current_user, db)
296 # Get latest task
297 latest_task = crud_deployments.get_latest_task(db, deployment_id)
298 task_summary = None
299 logs = None
301 if latest_task:
302 task_summary = TaskSummary(
303 taskId=latest_task.taskId,
304 type=latest_task.type,
305 status=latest_task.status,
306 started_at=latest_task.started_at,
307 finished_at=latest_task.finished_at,
308 created_at=latest_task.created_at,
309 current_phase=getattr(latest_task, "current_phase", None),
310 progress_pct=getattr(latest_task, "progress_pct", None),
311 )
312 if include_logs:
313 logs = latest_task.logs
315 # Get teams with members. The owner view sees every team and
316 # every member. The member view only sees their own team(s) so
317 # they can't browse who else has access to the deployment.
318 # ``can_view_deployment_owner`` widens "owner view" to course-teachers
319 # of the deployment-owner's course, so they get the full roster + logs
320 # + outputs alongside owners and admins.
321 is_owner_view = can_view_deployment_owner(current_user, deployment, db)
322 teams_data = crud_deployments.get_deployment_teams_with_members(db, deployment_id)
323 if not is_owner_view:
324 teams_data = [
325 t for t in teams_data
326 if any(str(m["userId"]) == str(current_user.userId) for m in t["members"])
327 ]
328 teams = [
329 DeploymentTeamResponse(
330 teamId=team["teamId"],
331 name=team["name"],
332 members=[
333 DeploymentTeamMember(
334 userId=member["userId"],
335 email=member["email"],
336 username=member["username"]
337 )
338 for member in team["members"]
339 ]
340 )
341 for team in teams_data
342 ]
344 # Outputs / state / logs are owner-only — members don't get to
345 # browse the credentials of teammates or the raw infrastructure
346 # state. They have their own resend-access action for their own
347 # credentials.
348 if is_owner_view:
349 outputs_data = crud_deployments.get_deployment_outputs(db, deployment_id)
350 outputs = DeploymentOutputs(raw=outputs_data) if outputs_data else None
351 else:
352 outputs = None
353 logs = None
355 # Get status and created_at from tasks
356 status_value = crud_deployments.get_deployment_status(db, deployment_id)
357 created_at = crud_deployments.get_deployment_created_at(db, deployment_id)
359 # Parse userInputVar JSON string back to dict if it exists. Same
360 # strip-file-bytes treatment as the list endpoint — base64
361 # payloads are surfaced via the download route, not the JSON view.
362 user_input_var_parsed = _parse_and_strip_user_input(deployment.userInputVar)
364 # ``deployment.app`` is the raw ORM relation whose ``image`` column
365 # carries bytes. Pydantic's ``DeploymentDetail`` declares
366 # ``app.image: Optional[str]`` (the wire shape is a ``data:image/...``
367 # URL), so handing it the bytes verbatim throws ``string_unicode``.
368 # Run it through ``_serialize_app`` — the same helper the
369 # ``/apps``-endpoints already use — to swap the bytes for the
370 # data-URL string in place. Apps without an uploaded image are
371 # unaffected (``getattr`` returns ``None`` and the helper no-ops).
372 from app.routers.apps import _serialize_app # local: avoid import cycle
373 serialised_app = _serialize_app(deployment.app)
375 return DeploymentDetail(
376 deploymentId=deployment.deploymentId,
377 name=deployment.name,
378 appId=deployment.appId,
379 userId=deployment.userId,
380 releaseTag=deployment.releaseTag,
381 userInputVar=user_input_var_parsed,
382 status=status_value,
383 created_at=created_at,
384 user=deployment.user,
385 app=serialised_app,
386 teams=teams,
387 latest_task=task_summary,
388 outputs=outputs,
389 logs=logs,
390 )
393# ----------------------------------------------------------------
394# CREATE DEPLOYMENT
395# ----------------------------------------------------------------
397# Defense-in-depth limits for inline file uploads. The UX-side warning
398# is mirrored on the wizard, but a hand-crafted POST could still try
399# to push GBs of payload through ``userInputVar``. We refuse before
400# the row hits the DB.
401#
402# Per-file cap matches the existing app-image cap so users don't have
403# to learn a second number; deployment-wide cap is 5× that, leaving
404# headroom for (e.g.) one big assignment plus several small starter
405# files. Both are enforced post-base64-decode so a malicious base64
406# blob of right-shape but wrong-size still fails fast.
407_MAX_FILE_BYTES_PER_FILE = 2 * 1024 * 1024
408_MAX_FILE_BYTES_PER_DEPLOYMENT = 10 * 1024 * 1024
411def _attach_files_to_user_input(
412 user_input_var: dict | None,
413 files: dict | None,
414 variable_definitions: list[dict] | None = None,
415) -> dict:
416 """Validate and merge wizard-uploaded files into ``userInputVar``.
418 The wizard ships files in a parallel ``files`` field instead of
419 nesting them straight into ``userInputVar.terraform`` so the
420 request payload's shape is obvious to a reader and so we can
421 apply size / encoding validation in one place. Result is a fresh
422 dict with the files folded into ``terraform[var_name]`` — the
423 worker doesn't need to know they originally came from a separate
424 field.
426 Validation:
427 * each top-level key in ``files`` becomes one terraform variable
428 * each inner-map entry is one ``DeploymentFileUpload`` record
429 * ``content_b64`` decodes cleanly (RFC 4648, padding optional)
430 * decoded size matches the declared ``size`` (within rounding —
431 client may have set it before encoding so we accept ±1)
432 * per-file cap and total deployment cap
433 * if ``variable_definitions`` are provided and a file variable
434 declares ``fileExtensions``, each uploaded filename's suffix
435 (lowercased, after the last dot) must be in the allowed list.
436 Defense-in-depth: the wizard's ``accept`` attribute already
437 filters in the picker, but a hand-crafted POST could bypass it.
439 Raises ``HTTPException(413)`` for size violations and
440 ``HTTPException(422)`` for malformed payload — Pydantic already
441 rejected the obvious cases (missing fields, wrong types) before
442 we get here, so we only catch what gets past it.
443 """
444 base = dict(user_input_var or {})
445 base.setdefault("terraform", {})
446 base.setdefault("packer", {})
448 if not files:
449 return base
451 # Build an index var_name → allowed_extensions for the extension
452 # check below. Variables without ``fileExtensions`` skip the
453 # filter — keeps backward compatibility for any caller that doesn't
454 # supply ``variable_definitions``.
455 allowed_exts_by_var: dict[str, list[str]] = {}
456 scoped_file_vars: set[str] = set()
457 if variable_definitions:
458 for vdef in variable_definitions:
459 exts = vdef.get("fileExtensions")
460 if exts:
461 allowed_exts_by_var[vdef["name"]] = [e.lower() for e in exts]
462 if vdef.get("varScope") in ("team", "user"):
463 scoped_file_vars.add(vdef["name"])
465 total_bytes = 0
466 terraform_block = dict(base.get("terraform") or {})
468 for var_name, slot_map in files.items():
469 if var_name in terraform_block:
470 # Wizard already routed something into this variable — a
471 # collision means the frontend filled both the variables
472 # picker AND the file uploader for the same name. That's an
473 # unrecoverable contract violation; surface it clearly.
474 raise HTTPException(
475 status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
476 detail={
477 "reason": "file_var_collision",
478 "variable": var_name,
479 },
480 )
481 if not isinstance(slot_map, dict) or not slot_map:
482 raise HTTPException(
483 status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
484 detail={"reason": "file_var_empty", "variable": var_name},
485 )
487 encoded_slots: dict[str, dict] = {}
488 for slot_key, upload in slot_map.items():
489 # ``upload`` arrives here as a Pydantic model instance
490 # already (FastAPI deserialised the request body into
491 # ``DeploymentCreate``) — pull fields off attributes.
492 content_b64 = upload.content_b64
494 # Extension-filter check — only when the app author declared
495 # an ``@openstack:file:<scope>:<exts>`` filter. We compare
496 # the filename suffix (after the last dot, lowercased) to
497 # the allowed list. Missing dot or unknown suffix → 422.
498 allowed_exts = allowed_exts_by_var.get(var_name)
499 if allowed_exts is not None:
500 name = upload.name or ""
501 dot = name.rfind(".")
502 suffix = name[dot + 1 :].lower() if dot >= 0 else ""
503 if suffix not in allowed_exts:
504 raise HTTPException(
505 status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
506 detail={
507 "reason": "file_extension_rejected",
508 "variable": var_name,
509 "slot": slot_key,
510 "filename": upload.name,
511 "allowed": allowed_exts,
512 },
513 )
514 try:
515 # ``validate=True`` would reject any non-base64
516 # whitespace; the wizard sends compact base64 so this
517 # is fine.
518 decoded = base64.b64decode(content_b64, validate=True)
519 except (binascii.Error, ValueError) as e:
520 raise HTTPException(
521 status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
522 detail={
523 "reason": "file_b64_invalid",
524 "variable": var_name,
525 "slot": slot_key,
526 "error": str(e),
527 },
528 )
530 if abs(len(decoded) - upload.size) > 1:
531 raise HTTPException(
532 status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
533 detail={
534 "reason": "file_size_mismatch",
535 "variable": var_name,
536 "slot": slot_key,
537 "declared": upload.size,
538 "actual": len(decoded),
539 },
540 )
542 if len(decoded) > _MAX_FILE_BYTES_PER_FILE:
543 raise HTTPException(
544 status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
545 detail={
546 "reason": "file_too_large",
547 "variable": var_name,
548 "slot": slot_key,
549 "limit_bytes": _MAX_FILE_BYTES_PER_FILE,
550 "actual_bytes": len(decoded),
551 },
552 )
553 total_bytes += len(decoded)
554 if total_bytes > _MAX_FILE_BYTES_PER_DEPLOYMENT:
555 raise HTTPException(
556 status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE,
557 detail={
558 "reason": "deployment_files_too_large",
559 "limit_bytes": _MAX_FILE_BYTES_PER_DEPLOYMENT,
560 },
561 )
563 encoded_slots[slot_key] = {
564 "name": upload.name,
565 "content_b64": content_b64,
566 "size": upload.size,
567 "content_type": upload.content_type or "application/octet-stream",
568 }
569 # scope=team|user: HCL type is map(map(object({...}))) —
570 # outer key is the team/user slot, inner key is the upload slot.
571 # scope=all: HCL type is map(object({...})) — flat map.
572 if var_name in scoped_file_vars:
573 terraform_block[var_name] = {
574 slot_key: {"uploaded": file_obj}
575 for slot_key, file_obj in encoded_slots.items()
576 }
577 else:
578 terraform_block[var_name] = encoded_slots
580 base["terraform"] = terraform_block
581 return base
584def _validate_scoped_user_input(
585 user_input_var: dict | None,
586 variable_definitions: list[dict],
587 teams_payload: list,
588) -> None:
589 """Enforce that variables marked with ``varScope = team|user``
590 arrive as a map whose keys match the deployment's team / user roster.
592 Reasoning: the wizard packs scoped variables as a Map
593 (``{slot_key: value, ...}``) and ships them via ``userInputVar``.
594 A hand-crafted POST could ship arbitrary keys; we want unknown
595 Scope-Targets to fail fast and loud before they hit Terraform,
596 where the error would be a confusing "module: invalid for_each
597 key" deep in the worker log.
599 File variables are NOT skipped here — they share the same scoped
600 map shape (``{slot_key: file_obj}``) and a hand-crafted POST could
601 just as easily smuggle an unknown team name into a file-scope
602 variable. We validate slot identity against the same roster; the
603 per-file size / base64 / extension validation stays in
604 :func:`_attach_files_to_user_input` because that's the layer that
605 actually decodes the bytes.
607 Raises ``HTTPException(422)`` with ``reason="unknown_scope_target"``,
608 ``reason="scoped_var_not_map"``, or ``reason="required_slot_empty"``
609 for shape/identity/completeness problems.
610 """
611 if not user_input_var:
612 return
614 # Compose the universe of valid slot keys per scope. ``team``
615 # accepts any team name; ``user`` accepts ``TeamName-Username``
616 # composites — mirror of ``userSlotKey`` in the wizard.
617 team_names: set[str] = set()
618 for team in teams_payload or []:
619 team_name = getattr(team, "name", None) or (team.get("name") if isinstance(team, dict) else None)
620 if not team_name:
621 continue
622 team_names.add(team_name)
623 # ``team.userIds`` contains UUID strings here, not usernames —
624 # the deployment endpoint resolves usernames just below us
625 # when assembling ``teams_dict``. We accept any non-empty
626 # composite key prefix-matching ``f"{team_name}-"`` for
627 # user-scoped variables, because the wizard renders one slot
628 # per member and labels it with the username (not the UUID).
629 # A stricter check would require an extra DB round-trip; the
630 # prefix-and-non-empty check is enough to catch typos and
631 # cross-team key smuggling.
633 # Longest-prefix-match helper for user-scope composite keys:
634 # ``TeamName-Username``. A naive ``slot_key.find('-')`` would
635 # truncate a team named ``Team-A`` to just ``Team``, so any team
636 # name containing a dash would be misclassified as unknown. We
637 # iterate the known team names from longest to shortest and pick
638 # the first one that either equals ``slot_key`` (empty username,
639 # rejected below) or prefixes it as ``f"{team}-"``.
640 teams_by_length = sorted(team_names, key=len, reverse=True)
642 def _user_slot_team_prefix(slot_key: str) -> str | None:
643 for team in teams_by_length:
644 if slot_key == team:
645 # No trailing ``-Username`` — caller treats this as a
646 # missing-user-segment and surfaces ``unknown_scope_target``.
647 return team
648 if slot_key.startswith(team + "-"):
649 return team
650 return None
652 def _is_empty_slot_value(val) -> bool:
653 """Treat None, empty string, empty list, and empty dict as
654 "slot not filled". The wizard would otherwise let a required
655 team/user-scoped var slip through with one team left blank,
656 which Terraform would catch with a much less actionable
657 ``Inappropriate value for attribute`` deep in the worker log.
658 """
659 if val is None:
660 return True
661 if isinstance(val, str) and val == "":
662 return True
663 return isinstance(val, (list, dict)) and len(val) == 0
665 for source_key in ("terraform", "packer"):
666 block = user_input_var.get(source_key)
667 if not isinstance(block, dict):
668 continue
669 for vdef in variable_definitions:
670 if vdef.get("source") != source_key:
671 continue
672 scope = vdef.get("varScope")
673 if scope not in ("team", "user"):
674 continue
675 var_name = vdef["name"]
676 value = block.get(var_name)
677 is_file = vdef.get("osType") == "file"
678 required = bool(vdef.get("required"))
679 if value is None:
680 # File-scope vars MUST be present — the wizard always
681 # ships at least an empty map for them, so a None here
682 # is a hand-crafted-POST shape. For non-file required
683 # scoped vars, raise on the slot-completeness check
684 # below by treating the absent value as an empty map.
685 if required:
686 value = {}
687 else:
688 continue # variable left at HCL default — allowed
689 if not isinstance(value, dict):
690 raise HTTPException(
691 status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
692 detail={
693 "reason": "scoped_var_not_map",
694 "variable": var_name,
695 "scope": scope,
696 },
697 )
698 for slot_key in value:
699 if scope == "team":
700 if slot_key not in team_names:
701 raise HTTPException(
702 status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
703 detail={
704 "reason": "unknown_scope_target",
705 "variable": var_name,
706 "scope": scope,
707 "slot": slot_key,
708 "allowed": sorted(team_names),
709 },
710 )
711 else: # user scope
712 # Longest-prefix-match against known team names so
713 # a team named ``Team-A`` parses to prefix
714 # ``Team-A`` and rest ``Username`` instead of
715 # prefix ``Team`` (which wouldn't be a known team).
716 prefix = _user_slot_team_prefix(slot_key)
717 if prefix is None or slot_key == prefix:
718 raise HTTPException(
719 status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
720 detail={
721 "reason": "unknown_scope_target",
722 "variable": var_name,
723 "scope": scope,
724 "slot": slot_key,
725 "hint": "expected ``TeamName-Username``",
726 },
727 )
729 # Required slot-completeness check: for required team /
730 # user scoped variables every expected slot key must carry
731 # a non-empty value. Without this an empty map (or one
732 # team left blank) would silently pass here and only fail
733 # downstream with an opaque Terraform error.
734 #
735 # File vars are skipped from the completeness sweep — the
736 # per-file size/decode validation in
737 # :func:`_attach_files_to_user_input` raises a more specific
738 # error (file_var_empty / file_b64_invalid) for them. We
739 # only checked slot identity above; the bytes themselves
740 # are validated at that layer.
741 if required and not is_file:
742 expected_slots: set[str] = set()
743 if scope == "team":
744 expected_slots = set(team_names)
745 # For ``user`` scope we don't have the per-team member
746 # roster here (would need a DB round-trip we already
747 # avoid above), so we only enforce that each slot the
748 # caller did ship carries a non-empty value. The
749 # wizard's frontend check is the primary guard; this
750 # is defense-in-depth against hand-crafted POSTs that
751 # ship one half-filled team. A POST that omits a team
752 # entirely for a required user-scope var is caught by
753 # the team-scope branch via team_names because the
754 # wizard always emits at least one slot per team.
756 missing: list[str] = []
757 for slot in expected_slots:
758 if _is_empty_slot_value(value.get(slot)):
759 missing.append(slot)
760 # Also flag empty values among slots the caller did
761 # provide — covers user-scope and any partial-fill case.
762 for slot, val in value.items():
763 if _is_empty_slot_value(val) and slot not in missing:
764 missing.append(slot)
765 if missing:
766 raise HTTPException(
767 status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
768 detail={
769 "reason": "required_slot_empty",
770 "variable": var_name,
771 "scope": scope,
772 "missing_slots": sorted(missing),
773 },
774 )
777def _strip_file_vars_from_user_input(user_input_var: dict | None) -> dict | None:
778 """Strip per-file ``content_b64`` payloads from a userInputVar dict.
780 Used by the deployment detail responses so the JSON the frontend
781 receives only carries metadata (name/size/content_type) — the
782 decoded bytes can be many MBs each and shipping them on every
783 page render is wasteful. Owners who actually want the file fetch
784 it via the dedicated download endpoint.
786 Heuristic-based: a variable is a file slot when its value is a
787 mapping whose entries each carry a ``content_b64`` field — the
788 same shape ``_attach_files_to_user_input`` writes. We match on
789 that key because no other user-input kind uses it.
790 """
791 if not isinstance(user_input_var, dict):
792 return user_input_var
794 out = {k: v for k, v in user_input_var.items() if k != "terraform"}
795 tf_block = user_input_var.get("terraform")
796 if not isinstance(tf_block, dict):
797 if "terraform" in user_input_var:
798 out["terraform"] = tf_block
799 return out
801 stripped_tf: dict = {}
802 for var_name, value in tf_block.items():
803 if _looks_like_file_var(value):
804 stripped_tf[var_name] = _file_var_metadata_only(value)
805 else:
806 stripped_tf[var_name] = value
807 out["terraform"] = stripped_tf
808 return out
811def _parse_and_strip_user_input(raw: str | None) -> dict | None:
812 """Parse a stored ``userInputVar`` JSON string and strip file bytes.
814 Wraps the ``json.loads`` → :func:`_strip_file_vars_from_user_input`
815 chain (with a malformed-JSON guard) shared by the list, detail, and
816 create responses so all three surface the same file-stripped shape.
817 Returns ``None`` for an empty or unparseable value.
818 """
819 if not raw:
820 return None
821 try:
822 return _strip_file_vars_from_user_input(json.loads(raw))
823 except json.JSONDecodeError:
824 return None
827def _looks_like_file_var(value) -> bool:
828 """True if ``value`` matches the file-upload shape produced by
829 :func:`_attach_files_to_user_input`: a non-empty mapping whose
830 values are objects carrying ``content_b64`` plus the metadata
831 triplet. Used at response-shaping time to identify file-typed
832 variables without consulting the app's variable schema, AND at
833 lifecycle-dispatch time (destroy/pause/resume/redeploy) to drop
834 file vars from the worker's var-set so Terraform's schema
835 validation doesn't trip on a payload it doesn't need.
837 Shape examples it matches (and only these):
839 * ``scope=all`` → ``{"all": {name, content_b64, size, content_type}}``
840 * ``scope=team`` → ``{"Team-1": {...}, "Team-2": {...}}``
841 * ``scope=user`` → ``{"Team-1-luca": {...}, ...}``
843 Strict signature: each slot must carry ``content_b64``. Rows
844 that survived an earlier response-side-strip-then-persisted
845 accident (metadata triplet only, no bytes) are NOT auto-
846 detected — clean them up by hand (delete the deployment row +
847 its pg-backend tfstate schema). The strictness is intentional:
848 a lenient detector would silently swallow legitimate non-file
849 map variables that coincidentally share the metadata key names.
850 """
851 if not isinstance(value, dict) or not value:
852 return False
853 for slot in value.values():
854 if not isinstance(slot, dict):
855 return False
856 if "content_b64" not in slot:
857 return False
858 return True
861def _file_var_metadata_only(value: dict) -> dict:
862 """Return a copy of a file-shape variable with the ``content_b64``
863 payload stripped. Metadata fields (name, size, content_type)
864 survive so the UI can list "what was uploaded" without shipping
865 base64 megabytes on every detail-view render.
866 """
867 out: dict = {}
868 for slot_key, slot in value.items():
869 out[slot_key] = {k: v for k, v in slot.items() if k != "content_b64"}
870 return out
873# ----------------------------------------------------------------
874# CREATE DEPLOYMENT
875# ----------------------------------------------------------------
876@router.post("/", response_model=DeploymentResponse, status_code=status.HTTP_201_CREATED)
877def create_deployment(
878 deployment: DeploymentCreate,
879 db: Session = Depends(get_db),
880 current_user: User = Depends(get_current_user_keycloak)
881):
882 """
883 Create a new deployment
885 Atomicity: a per-user advisory lock serializes credential mutation
886 with deployment dispatch. The deployment row, teams, user mappings,
887 and the initial PENDING task row are all inserted in a single
888 transaction, so the user can never end up with a deployment row
889 that has no matching task. Celery dispatch happens AFTER commit;
890 if it fails, the task row is flipped to FAILED so the deployment
891 surfaces an honest error instead of hanging in PENDING forever.
892 """
893 # Per-user lock — serializes against PUT /me/openstack-credentials
894 # and any other concurrent POST /deployments from this user. Held
895 # until the next COMMIT/ROLLBACK on this connection.
896 crud_locks.acquire_user_xact_lock(db, current_user.userId)
898 # Refuse the create if the target app is soft-deleted.
899 target_app = crud_apps.get_app(db, deployment.appId)
900 if target_app is None:
901 raise HTTPException(
902 status_code=status.HTTP_404_NOT_FOUND,
903 detail={"reason": "app_not_found_or_deleted"},
904 )
906 # Gate the create on the same visibility rule the list/detail
907 # endpoints use. A student cannot deploy a private app they don't
908 # own, and a non-owner cannot deploy an app without an approved
909 # version. The owner / admin path stays open.
910 # ``ensure_view_app`` raises 403 with the structured payload, so
911 # the frontend receives the same shape it sees on the detail
912 # endpoint when visibility is denied.
913 ensure_view_app(current_user, target_app, db=db)
915 # Load the app author's variable declarations so we can enforce
916 # per-variable contracts (``varScope``, ``fileExtensions``) below.
917 # We only do this when the request actually carries variables /
918 # files — for a no-input deploy the round-trip into Git would be
919 # waste. Same parser as ``GET /apps/{id}/variables`` so client and
920 # server agree on which variable is scoped/file/free-text.
921 variable_definitions: list[dict] = []
922 if deployment.userInputVar or deployment.files:
923 from app.routers.apps import load_variable_definitions
924 release_tag = deployment.releaseTag or "main"
925 try:
926 variable_definitions = load_variable_definitions(target_app, release_tag)
927 except HTTPException as exc:
928 if exc.status_code == status.HTTP_422_UNPROCESSABLE_ENTITY:
929 raise
930 # 400 (no git_link) or 500 (git unreachable) — skip validation,
931 # proceed without variable definitions.
933 # Fold the wizard's parallel ``files`` upload into
934 # ``userInputVar.terraform`` before the row gets persisted, so the
935 # rest of this handler — and the worker downstream — sees one
936 # uniform dict. The helper validates base64 / size / per-file and
937 # per-deployment caps; any failure short-circuits with a 4xx and
938 # the row never enters the DB. When variable definitions are
939 # available, the helper also enforces the app author's declared
940 # ``fileExtensions`` filter on each upload name.
941 deployment.userInputVar = _attach_files_to_user_input(
942 deployment.userInputVar, deployment.files, variable_definitions or None,
943 )
945 # Enforce ``varScope = team|user`` contracts: each value must be a
946 # map whose keys match the deployment's team / user roster. This
947 # is defense-in-depth — the wizard already only renders slots the
948 # user can fill, but a hand-crafted POST could ship unknown keys.
949 if variable_definitions:
950 _validate_scoped_user_input(
951 deployment.userInputVar,
952 variable_definitions,
953 deployment.teams or [],
954 )
956 db_deployment = crud_deployments.create_deployment(
957 db, deployment, current_user.userId
958 )
960 user_ids_in_deployment = set()
961 if deployment.teams:
962 teams_data = [
963 {"name": team.name, "userIds": team.userIds}
964 for team in deployment.teams
965 ]
966 crud_teams.create_teams_for_deployment(
967 db=db,
968 deployment_id=db_deployment.deploymentId,
969 teams_data=teams_data,
970 )
971 for team in deployment.teams:
972 user_ids_in_deployment.update(team.userIds)
974 if user_ids_in_deployment:
975 crud_deployments.create_user_to_deployments(
976 db=db,
977 deployment_id=db_deployment.deploymentId,
978 user_ids=user_ids_in_deployment,
979 )
981 # Parse user input variables
982 try:
983 user_vars = (
984 json.loads(db_deployment.userInputVar) if db_deployment.userInputVar else {}
985 )
986 except Exception:
987 user_vars = {}
989 # Format teams for Terraform (team_name: [user_emails])
990 teams_dict = {}
991 if deployment.teams:
992 for team in deployment.teams:
993 team_users = []
994 for user_id in team.userIds:
995 user = crud_users.get_user(db, user_id)
996 if user:
997 team_users.append({"email": user.email})
998 teams_dict[team.name] = team_users
1000 # Per-user OpenStack credentials are required to deploy. The envelope
1001 # carries ciphertext only — the worker decrypts in-process. Reading
1002 # this inside the locked TX guarantees the envelope matches whatever
1003 # credential row a concurrent PUT might have committed: PUT is
1004 # serialized behind us by the same advisory lock.
1005 openstack_envelope = _fetch_dispatch_envelope(
1006 db, current_user.userId, rollback_on_missing=True
1007 )
1009 # Insert PENDING task row in the SAME transaction as the deployment,
1010 # commit atomically (deployment + teams + user_to_deployments + task),
1011 # then dispatch to Celery outside the locked TX. On a send failure the
1012 # task row is flipped to FAILED and we surface 503 — the deployment
1013 # row stays, but the user sees an obvious failure instead of an
1014 # eternal PENDING.
1015 _commit_and_dispatch(
1016 db,
1017 deployment_id=db_deployment.deploymentId,
1018 task_type=TaskType.DEPLOY,
1019 celery_task_name="tasks.deploy_application",
1020 celery_args=[
1021 str(db_deployment.deploymentId),
1022 str(db_deployment.appId),
1023 db_deployment.app.git_link,
1024 db_deployment.releaseTag,
1025 user_vars,
1026 teams_dict,
1027 openstack_envelope,
1028 ],
1029 dispatch_error_label="Could not dispatch deployment task — please retry",
1030 rollback_on_conflict=True,
1031 )
1033 db.refresh(db_deployment)
1035 status_value = crud_deployments.get_deployment_status(db, db_deployment.deploymentId)
1036 created_at = crud_deployments.get_deployment_created_at(db, db_deployment.deploymentId)
1038 # Same file-strip rule as the list/detail endpoints: the POST
1039 # response shape mirrors the read shape so the frontend can reuse
1040 # the same parsing code-path.
1041 user_input_var_parsed = _parse_and_strip_user_input(db_deployment.userInputVar)
1043 return DeploymentResponse(
1044 deploymentId=db_deployment.deploymentId,
1045 name=db_deployment.name,
1046 appId=db_deployment.appId,
1047 userId=db_deployment.userId,
1048 releaseTag=db_deployment.releaseTag,
1049 userInputVar=user_input_var_parsed,
1050 status=status_value,
1051 created_at=created_at,
1052 )
1055# ----------------------------------------------------------------
1056# DELETE DEPLOYMENT
1057# ----------------------------------------------------------------
1058#
1059# One endpoint, two outcomes — the backend picks the right one from
1060# the deployment's status:
1061#
1062# * ``success`` / ``failed`` / ``paused`` → dispatch a Destroy task
1063# (terraform destroy + auto-soft-delete on success). ``paused`` is
1064# in the destroy set because SHUTOFF instances + volumes/networks
1065# are still OpenStack resources that need to be reclaimed.
1066# Returns 202 + task_id; the frontend keeps the live stream open
1067# and routes back to the list when the task finishes.
1068# * ``cancelled`` → soft-delete immediately. Returns 204.
1069# * any other status (running / pending / destroying / pausing / resuming)
1070# → 409, the user has to wait.
1071#
1072# Frontend doesn't have to know the difference — it just calls DELETE
1073# and switches into the live-stream view when the response is 202.
1074@router.delete("/{deployment_id}")
1075def delete_deployment(
1076 deployment_id: UUID,
1077 db: Session = Depends(get_db),
1078 current_user: User = Depends(get_current_user_keycloak),
1079):
1080 """Unified delete — destroys OpenStack resources first if needed.
1082 Restricted to the owner-view (creator, teacher, admin). Members
1083 can read-access the deployment but never tear it down.
1084 """
1085 deployment = crud_deployments.get_deployment_with_details(db, deployment_id)
1086 if not deployment:
1087 raise HTTPException(
1088 status_code=status.HTTP_404_NOT_FOUND,
1089 detail="Deployment not found",
1090 )
1092 # Destructive operation — uses the operate gate, which is
1093 # owner-or-admin only. Course-teachers explicitly do NOT get
1094 # delete/destroy rights on deployments in their courses; they only
1095 # get inspect (logs, infra).
1096 ensure_operate_deployment(current_user, deployment, db)
1098 # Per-deployment advisory lock — serialises against any concurrent
1099 # POST /pause, /resume or DELETE on the same deployment so the
1100 # ``current_status`` read below and the eventual
1101 # ``prepare_task_in_tx`` insert see a consistent picture. Without
1102 # this, two concurrent destroys could both pass the in-flight check
1103 # and one would crash on the partial unique index.
1104 crud_locks.acquire_deployment_xact_lock(db, deployment_id)
1106 current_status = crud_deployments.get_deployment_status(db, deployment_id)
1108 # Active task in flight — neither path is safe.
1109 if current_status in lifecycle_service.IN_FLIGHT_STATUSES:
1110 raise HTTPException(
1111 status_code=status.HTTP_409_CONFLICT,
1112 detail=(
1113 f"Cannot delete a deployment in status '{current_status}'. "
1114 "Wait for the active task to finish."
1115 ),
1116 )
1118 # Resources may exist — destroy them first; the listener will
1119 # auto-soft-delete the row when the destroy task succeeds. ``paused``
1120 # also lands here: SHUTOFF instances + volumes/networks are still
1121 # OpenStack resources that need to be torn down before the row can
1122 # be hidden. ``pause_failed`` / ``resume_failed`` likewise still
1123 # have running OpenStack resources behind them — the deployment
1124 # itself didn't break, only the lifecycle pass.
1125 if current_status in ("success", "failed", "paused", "pause_failed", "resume_failed"):
1126 return _dispatch_destroy(db, deployment, current_user)
1128 # No resources to clean up (cancelled, or anything else terminal):
1129 # straight soft-delete.
1130 success = crud_deployments.soft_delete_deployment(db, deployment_id)
1131 if not success:
1132 raise HTTPException(
1133 status_code=status.HTTP_404_NOT_FOUND,
1134 detail="Deployment not found",
1135 )
1136 return Response(status_code=status.HTTP_204_NO_CONTENT)
1139def _fetch_dispatch_envelope(db: Session, user_id, *, rollback_on_missing: bool):
1140 """Fetch the encrypted OpenStack credential envelope for a dispatch.
1142 Raises ``HTTPException(412, openstack_credentials_missing)`` when the
1143 user has no credentials. ``rollback_on_missing`` is used by the
1144 create path, which holds an uncommitted deployment insert that must
1145 be rolled back before the 412 escapes; lifecycle callers have no such
1146 pending insert and pass ``False``.
1147 """
1148 try:
1149 return crud_openstack_credentials.get_dispatch_envelope(db, user_id)
1150 except crud_openstack_credentials.NoCredentialError:
1151 if rollback_on_missing:
1152 db.rollback()
1153 raise HTTPException(
1154 status_code=status.HTTP_412_PRECONDITION_FAILED,
1155 detail={"reason": "openstack_credentials_missing"},
1156 )
1159def _commit_and_dispatch(
1160 db: Session,
1161 *,
1162 deployment_id,
1163 task_type: TaskType,
1164 celery_task_name: str,
1165 celery_args: list,
1166 dispatch_error_label: str,
1167 rollback_on_conflict: bool,
1168):
1169 """Insert the PENDING task in-TX, commit, then dispatch to Celery.
1171 Shared tail of the create and lifecycle dispatch paths:
1173 * ``prepare_task_in_tx`` → ``HTTPException(409)`` on an active task
1174 (the create path additionally rolls back its pending insert);
1175 * ``commit`` + ``refresh`` the task row;
1176 * ``dispatch_to_celery`` → ``HTTPException(503, dispatch_error_label)``
1177 on a Celery send failure (the task row is flipped to FAILED inside
1178 ``dispatch_to_celery`` in a fresh TX).
1180 Returns the committed, dispatched ``Task``.
1181 """
1182 try:
1183 task = task_service_module.prepare_task_in_tx(
1184 db,
1185 deployment_id=deployment_id,
1186 task_type=task_type,
1187 )
1188 except task_service_module.ActiveTaskExistsError:
1189 if rollback_on_conflict:
1190 db.rollback()
1191 raise HTTPException(
1192 status_code=status.HTTP_409_CONFLICT,
1193 detail="Deployment already has an active task",
1194 )
1196 db.commit()
1197 db.refresh(task)
1199 try:
1200 task, _celery_id = task_service_module.dispatch_to_celery(
1201 db,
1202 task=task,
1203 celery_task_name=celery_task_name,
1204 celery_args=celery_args,
1205 )
1206 except Exception:
1207 raise HTTPException(
1208 status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
1209 detail=dispatch_error_label,
1210 )
1211 return task
1214def _dispatch_destroy(db: Session, deployment, current_user: User):
1215 """Enqueue the destroy worker task for a deployment.
1217 Thin wrapper around :func:`_dispatch_lifecycle_task` so DELETE can
1218 keep its existing two-call sites (success-path / failed-path) clear.
1219 """
1220 return _dispatch_lifecycle_task(
1221 db,
1222 deployment,
1223 current_user,
1224 task_type=TaskType.DESTROY,
1225 celery_task_name="tasks.destroy_deployment",
1226 response_status="destroying",
1227 )
1230def _dispatch_lifecycle_task(
1231 db: Session,
1232 deployment,
1233 current_user: User,
1234 task_type: TaskType,
1235 celery_task_name: str,
1236 response_status: str,
1237 extra_args: list | None = None,
1238):
1239 """Enqueue any post-deploy lifecycle worker task for a deployment.
1241 Used by destroy, pause, resume, and per-VM redeploy — all four
1242 follow the same pattern: load user inputs, gather team membership,
1243 fetch the encrypted OpenStack envelope, atomically insert a
1244 PENDING task row, commit, then ``send_task`` outside the locked
1245 TX. On a Celery send failure the task row flips to FAILED in a
1246 fresh TX (handled inside ``dispatch_to_celery``) and we surface a
1247 503 so the user sees an obvious failure instead of a permanent
1248 in-flight status.
1250 Args:
1251 task_type: the ``TaskType`` enum value that drives both
1252 the task row's ``type`` column and the
1253 status the partial-unique index prevents
1254 from coexisting.
1255 celery_task_name: name registered on the worker side (e.g.
1256 ``tasks.pause_deployment``).
1257 response_status: synthetic deployment status returned to the
1258 frontend in the 202 body — frontend uses
1259 this to immediately switch the UI into the
1260 live-stream view without re-fetching.
1261 extra_args: additional positional args appended to the
1262 Celery payload after the standard seven.
1263 Used by ``tasks.redeploy_resource`` to pass
1264 the targeted resource address.
1265 """
1266 try:
1267 user_vars = json.loads(deployment.userInputVar) if deployment.userInputVar else {}
1268 except Exception:
1269 user_vars = {}
1271 # Belt + braces: for lifecycle tasks that do NOT recreate the VM
1272 # (destroy, pause, resume), strip any ``@openstack:file:*`` payloads
1273 # from the user-vars BEFORE they reach the worker. Files are only
1274 # consumed at apply-time by cloud-init's write_files; everything
1275 # else just hands the same var-set to Terraform which then
1276 # validates the entire variable surface against the HCL schema.
1277 # A row whose ``content_b64`` was stripped by a response-side
1278 # ``_strip_file_vars_from_user_input`` pass (e.g. after a manual
1279 # DB edit, an in-place row shrink, or any future code path that
1280 # rewrites the persisted JSON) would otherwise crash destroy with
1281 # ``element "all": attributes "content_b64", "content_type",
1282 # "name", and "size" are required`` because the surviving slot
1283 # violates the variable's object type. Dropping the var
1284 # altogether lets Terraform fall back on the HCL default.
1285 #
1286 # REDEPLOY is the special case: ``terraform apply -replace`` destroys
1287 # and recreates the VM, so cloud-init runs fresh and MUST receive the
1288 # original ``write_files`` payload — otherwise the replaced VM ends
1289 # up empty even though the user/group/password config is preserved.
1290 # We therefore keep the file vars on REDEPLOY and let the persisted
1291 # base64 payload flow through to the worker, mirroring the initial
1292 # deploy path.
1293 #
1294 # DEPLOY/UPDATE legitimately need the file bytes too — they don't
1295 # enter the worker via this helper.
1296 if task_type in (TaskType.DESTROY, TaskType.PAUSE, TaskType.RESUME):
1297 terraform_block = user_vars.get("terraform")
1298 if isinstance(terraform_block, dict):
1299 user_vars = {
1300 **user_vars,
1301 "terraform": {
1302 k: v
1303 for k, v in terraform_block.items()
1304 if not _looks_like_file_var(v)
1305 },
1306 }
1308 teams_dict: dict = {}
1309 if deployment.teams:
1310 # Persisted Team rows expose membership via the ``user_to_teams``
1311 # association, not a flat ``userIds`` field — that lives on the
1312 # request-side Pydantic schema in the create endpoint, not on
1313 # the ORM. ``get_team_members`` does the join for us.
1314 for team in deployment.teams:
1315 members = crud_deployments.get_team_members(db, team.teamId)
1316 teams_dict[team.name] = [{"email": m.email} for m in members]
1318 openstack_envelope = _fetch_dispatch_envelope(
1319 db, current_user.userId, rollback_on_missing=False
1320 )
1322 task = _commit_and_dispatch(
1323 db,
1324 deployment_id=deployment.deploymentId,
1325 task_type=task_type,
1326 celery_task_name=celery_task_name,
1327 celery_args=[
1328 str(deployment.deploymentId),
1329 str(deployment.appId),
1330 deployment.app.git_link,
1331 deployment.releaseTag,
1332 user_vars,
1333 teams_dict,
1334 openstack_envelope,
1335 *(extra_args or []),
1336 ],
1337 dispatch_error_label=f"Could not dispatch {task_type.value} task — please retry",
1338 rollback_on_conflict=False,
1339 )
1341 return JSONResponse(
1342 status_code=status.HTTP_202_ACCEPTED,
1343 content={"task_id": str(task.taskId), "status": response_status},
1344 )
1347# ----------------------------------------------------------------
1348# INFRASTRUCTURE RESOURCES (per-deployment status + per-VM redeploy)
1349# ----------------------------------------------------------------
1350#
1351# Three sibling endpoints power the Infrastructure tab on the
1352# deployment detail page:
1353#
1354# * GET /{deployment_id}/resources?refresh=…
1355# Stage-1 listing — parses the cached TF state and (default)
1356# overlays live OpenStack lifecycle/hardware/addresses per VM.
1357# Returns a flat list spanning compute, network, subnet, SG,
1358# FIP, and port categories.
1359#
1360# * GET /{deployment_id}/resources/{address}
1361# Stage-2 detail — same shape, plus ports/SG-summary/volumes/
1362# metadata for ONE compute instance, identified by its TF state
1363# address (e.g. ``openstack_compute_instance_v2.team_ide["Team-A"]``).
1364# Frontend loads this lazily when the user opens a card's drawer.
1365#
1366# * POST /{deployment_id}/resources/{address}/redeploy
1367# Per-VM redeploy — issues ``terraform apply -replace=<addr>
1368# -target=<addr>`` in a dedicated Celery task. Strictly
1369# address-whitelisted against the cached TF state and the
1370# compute-instance category, so a hand-crafted POST can't smuggle
1371# a network-resource target (which would tear down all team VMs).
1372#
1373# All three are owner-only — the data exposed (live OpenStack status,
1374# the ability to bounce a VM) is not something a teammate should be
1375# able to access through the deployment detail page.
1378# We accept the same Terraform address vocabulary the user would type
1379# on ``terraform apply -target=``: ``type.name`` with optional
1380# ``[<int>]`` or ``["<string>"]`` suffix. Multiple address segments
1381# (modules, nested resources) aren't supported by the current apps,
1382# so we keep the regex strict to make smuggling impossible. The
1383# resource-existence whitelist below is the real defense; the regex
1384# is just a fast no-op rejection for obviously bad inputs (e.g.
1385# pipes, semicolons, spaces).
1386_TF_ADDRESS_RE = re.compile(
1387 r"""^
1388 [A-Za-z_][A-Za-z0-9_]* # provider type (e.g. openstack_compute_instance_v2)
1389 \.[A-Za-z_][A-Za-z0-9_-]* # resource name (e.g. team_ide)
1390 (?:
1391 \[(?:\d+|"[^"\\]+")\] # optional index ([0] or ["Team-A"])
1392 )?
1393 $""",
1394 re.VERBOSE,
1395)
1398def _latest_tf_state_for(deployment_id: UUID, db: Session) -> str | None:
1399 """Return the JSON blob of the most recent task that captured a
1400 Terraform state for this deployment, or None when no apply ever
1401 succeeded yet.
1403 Note: we deliberately do NOT filter by ``task.type`` — the worker
1404 captures state on deploy / destroy / pause / resume / redeploy
1405 alike, and any of those produce a valid snapshot. The "most
1406 recent" task wins, mirroring the existing ``get_deployment_outputs``
1407 semantics in ``crud/deployments.py``.
1408 """
1409 task = (
1410 db.query(TaskModel)
1411 .filter(TaskModel.deploymentId == deployment_id)
1412 .filter(TaskModel.tf_state.isnot(None))
1413 .order_by(desc(TaskModel.created_at))
1414 .first()
1415 )
1416 return task.tf_state if task else None
1419@router.get(
1420 "/{deployment_id}/resources",
1421 response_model=DeploymentResourceListResponse,
1422)
1423def list_deployment_resources(
1424 deployment_id: UUID,
1425 refresh: bool = True,
1426 db: Session = Depends(get_db),
1427 current_user: User = Depends(get_current_user_keycloak),
1428):
1429 """Stage-1 resource listing for the Infrastructure tab.
1431 Owner-only. ``refresh=false`` skips the live OpenStack join — use
1432 that when polling rapidly to avoid hammering Keystone, or when
1433 OpenStack is known unavailable and the cached state is good enough.
1434 """
1435 deployment = crud_deployments.get_deployment(db, deployment_id)
1436 if not deployment:
1437 raise HTTPException(
1438 status_code=status.HTTP_404_NOT_FOUND,
1439 detail="Deployment not found",
1440 )
1441 # Inspect-only view — owner, admin, or a course-teacher of the
1442 # deployment-owner's course. Course-teachers explicitly do NOT have
1443 # operate rights; this endpoint is read-only.
1444 ensure_view_deployment_owner(current_user, deployment, db)
1446 state_json = _latest_tf_state_for(deployment_id, db)
1447 views = build_resource_views(
1448 db=db,
1449 user=current_user,
1450 tf_state_json=state_json,
1451 refresh=refresh,
1452 )
1453 # Convert dataclasses → Pydantic models via dict-roundtrip. The
1454 # fields line up 1:1 by name, so ``model_validate`` works directly
1455 # on the dataclass dict.
1456 payload = [
1457 DeploymentResourceSchema.model_validate(_view_asdict(v))
1458 for v in views
1459 ]
1460 return DeploymentResourceListResponse(resources=payload, live=refresh)
1463@router.get(
1464 "/{deployment_id}/resources/{address:path}",
1465 response_model=DeploymentResourceSchema,
1466)
1467def get_deployment_resource_detail(
1468 deployment_id: UUID,
1469 address: str,
1470 db: Session = Depends(get_db),
1471 current_user: User = Depends(get_current_user_keycloak),
1472):
1473 """Stage-2 detail for one compute instance.
1475 Uses a ``path``-converter on the address so the for_each-key
1476 quoting (``team_ide["Team-A"]``) survives URL routing without
1477 aggressive encoding gymnastics on the client side. The address
1478 MUST exist in the cached state and MUST be a compute instance;
1479 other categories get 422.
1480 """
1481 deployment = crud_deployments.get_deployment(db, deployment_id)
1482 if not deployment:
1483 raise HTTPException(
1484 status_code=status.HTTP_404_NOT_FOUND,
1485 detail="Deployment not found",
1486 )
1487 # Inspect-only view — course-teachers may read the per-resource
1488 # detail; the per-VM redeploy below is operate-gated.
1489 ensure_view_deployment_owner(current_user, deployment, db)
1491 if not _TF_ADDRESS_RE.match(address):
1492 raise HTTPException(
1493 status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
1494 detail={"reason": "invalid_resource_address"},
1495 )
1497 state_json = _latest_tf_state_for(deployment_id, db)
1498 view = build_resource_detail(
1499 db=db,
1500 user=current_user,
1501 tf_state_json=state_json,
1502 address=address,
1503 )
1504 if view is None:
1505 raise HTTPException(
1506 status_code=status.HTTP_404_NOT_FOUND,
1507 detail={"reason": "resource_not_in_state", "address": address},
1508 )
1509 return DeploymentResourceSchema.model_validate(_view_asdict(view))
1512@router.post(
1513 "/{deployment_id}/resources/{address:path}/redeploy",
1514 status_code=status.HTTP_202_ACCEPTED,
1515)
1516def redeploy_deployment_resource(
1517 deployment_id: UUID,
1518 address: str,
1519 db: Session = Depends(get_db),
1520 current_user: User = Depends(get_current_user_keycloak),
1521):
1522 """Replace one compute instance via ``terraform apply -replace=…``.
1524 Address-whitelisted: we re-parse the cached TF state and only
1525 accept addresses that resolve to a compute instance. Anything else
1526 fails with 422 — the redeploy of a network resource would tear
1527 down all the team VMs, which is not what a one-VM "fix it" action
1528 should do.
1530 Concurrency: same per-user advisory lock as create/destroy/pause
1531 so a parallel redeploy can't race a destroy. The lock is held
1532 only for the row insert; Celery dispatch happens after commit.
1533 """
1534 crud_locks.acquire_user_xact_lock(db, current_user.userId)
1536 deployment = crud_deployments.get_deployment(db, deployment_id)
1537 if not deployment:
1538 raise HTTPException(
1539 status_code=status.HTTP_404_NOT_FOUND,
1540 detail="Deployment not found",
1541 )
1542 # Per-VM redeploy is a mutating operation — operate gate
1543 # (owner-or-admin). Course-teachers may inspect the resource via
1544 # the GET endpoints above but not bounce it.
1545 ensure_operate_deployment(current_user, deployment, db)
1547 if not _TF_ADDRESS_RE.match(address):
1548 raise HTTPException(
1549 status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
1550 detail={"reason": "invalid_resource_address"},
1551 )
1553 # Whitelist check: the address MUST point to a compute instance in
1554 # the current state. We re-parse here instead of trusting the
1555 # output of the list endpoint — a hand-crafted POST would skip
1556 # the list call entirely.
1557 state_json = _latest_tf_state_for(deployment_id, db)
1558 parsed = parse_tf_state(state_json)
1559 match = next((r for r in parsed if r.address == address), None)
1560 if match is None:
1561 raise HTTPException(
1562 status_code=status.HTTP_404_NOT_FOUND,
1563 detail={"reason": "resource_not_in_state", "address": address},
1564 )
1565 if match.category != "instance":
1566 raise HTTPException(
1567 status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
1568 detail={
1569 "reason": "non_redeployable_resource_type",
1570 "address": address,
1571 "category": match.category,
1572 },
1573 )
1575 return _dispatch_lifecycle_task(
1576 db=db,
1577 deployment=deployment,
1578 current_user=current_user,
1579 task_type=TaskType.REDEPLOY,
1580 celery_task_name="tasks.redeploy_resource",
1581 response_status="redeploying",
1582 extra_args=[address],
1583 )
1586def _view_asdict(view) -> dict:
1587 """Recursive dataclass → dict converter, used to bridge
1588 ``deployment_status.DeploymentResourceView`` to Pydantic.
1590 ``dataclasses.asdict`` already recurses into nested dataclasses,
1591 so we just delegate. Kept as a thin wrapper so the call sites
1592 above read symmetrically and we can swap in custom handling later
1593 if needed (e.g. enum serialisation).
1594 """
1595 return asdict(view)
1599#
1600# Halts the OpenStack compute instances of a deployment without
1601# tearing them down. The worker task pulls the terraform state, lists
1602# every ``openstack_compute_instance_v2`` resource, and runs
1603# ``openstack server stop`` against each. Volumes and networks stay,
1604# so RESUME restores the same instances byte-for-byte.
1605#
1606# Allowed only on ``status='success'`` — the lifecycle service is the
1607# single source of truth, the partial-unique index on active tasks is
1608# the DB-level backstop.
1609@router.post("/{deployment_id}/pause", status_code=status.HTTP_202_ACCEPTED)
1610def pause_deployment(
1611 deployment_id: UUID,
1612 db: Session = Depends(get_db),
1613 current_user: User = Depends(get_current_user_keycloak),
1614):
1615 """Pause a running deployment by stopping its compute instances.
1617 Owner-only — same gate as Destroy, because pausing a teammate's
1618 deployment is in practice a denial-of-service against the team.
1620 Returns ``202 + {task_id, status: "pausing"}`` on dispatch. The
1621 frontend reads ``status`` to switch to the live SSE view; the
1622 deployment's effective status is recomputed from the new task
1623 row by ``crud_deployments.get_deployment_status``.
1624 """
1625 deployment = crud_deployments.get_deployment_with_details(db, deployment_id)
1626 if not deployment:
1627 raise HTTPException(
1628 status_code=status.HTTP_404_NOT_FOUND,
1629 detail="Deployment not found",
1630 )
1632 ensure_operate_deployment(current_user, deployment, db)
1633 # Hold the per-deployment advisory lock across the status check
1634 # AND the task insert so a parallel POST /pause can't sneak past
1635 # ``ensure_action_allowed`` between our read and the
1636 # ``prepare_task_in_tx`` flush.
1637 crud_locks.acquire_deployment_xact_lock(db, deployment_id)
1638 lifecycle_service.ensure_action_allowed(
1639 db, deployment, lifecycle_service.DeploymentAction.PAUSE,
1640 )
1642 return _dispatch_lifecycle_task(
1643 db,
1644 deployment,
1645 current_user,
1646 task_type=TaskType.PAUSE,
1647 celery_task_name="tasks.pause_deployment",
1648 response_status="pausing",
1649 )
1652# ----------------------------------------------------------------
1653# RESUME DEPLOYMENT
1654# ----------------------------------------------------------------
1655#
1656# Reverses Pause. Allowed only on ``status='paused'`` — a deployment
1657# that was never paused has nothing to resume, so the lifecycle
1658# matrix gates this strictly. Returns 202 with the same shape as
1659# Pause/Destroy so the frontend handles all three the same way.
1660@router.post("/{deployment_id}/resume", status_code=status.HTTP_202_ACCEPTED)
1661def resume_deployment(
1662 deployment_id: UUID,
1663 db: Session = Depends(get_db),
1664 current_user: User = Depends(get_current_user_keycloak),
1665):
1666 """Resume a paused deployment by starting its compute instances."""
1667 deployment = crud_deployments.get_deployment_with_details(db, deployment_id)
1668 if not deployment:
1669 raise HTTPException(
1670 status_code=status.HTTP_404_NOT_FOUND,
1671 detail="Deployment not found",
1672 )
1674 ensure_operate_deployment(current_user, deployment, db)
1675 # Per-deployment advisory lock — same justification as in
1676 # ``pause_deployment`` above: keep the status check and the task
1677 # insert atomic against concurrent /resume / /pause / DELETE
1678 # requests on this deployment.
1679 crud_locks.acquire_deployment_xact_lock(db, deployment_id)
1680 lifecycle_service.ensure_action_allowed(
1681 db, deployment, lifecycle_service.DeploymentAction.RESUME,
1682 )
1684 return _dispatch_lifecycle_task(
1685 db,
1686 deployment,
1687 current_user,
1688 task_type=TaskType.RESUME,
1689 celery_task_name="tasks.resume_deployment",
1690 response_status="resuming",
1691 )
1694# ----------------------------------------------------------------
1695# DOWNLOAD UPLOADED FILE
1696# ----------------------------------------------------------------
1697#
1698# Lets the deployment owner re-fetch a file they uploaded at deploy
1699# time. The list / detail endpoints strip the base64 payload so they
1700# don't ship megabytes per page render; this endpoint is the only
1701# path that returns the actual bytes. Owner-only — members can see
1702# that a file was uploaded (metadata survives the strip), but the
1703# bytes themselves stay restricted to whoever created the deployment.
1704@router.get(
1705 "/{deployment_id}/files/{var_name}/{slot_key}",
1706 response_class=Response,
1707)
1708def download_deployment_file(
1709 deployment_id: UUID,
1710 var_name: str,
1711 slot_key: str,
1712 db: Session = Depends(get_db),
1713 current_user: User = Depends(get_current_user_keycloak),
1714):
1715 """Stream the raw bytes of one wizard-uploaded file back to the owner.
1717 Path components mirror how the upload was indexed:
1718 * ``var_name`` — the ``@openstack:file:*`` variable name
1719 * ``slot_key`` — the inner-map key (``"all"`` for scope=all,
1720 team name for scope=team, ``Team-User`` composite for scope=user)
1722 Returns 404 if any layer of the lookup misses; the frontend can
1723 therefore probe a slot's existence via this endpoint without
1724 needing a separate metadata response.
1725 """
1726 deployment = crud_deployments.get_deployment(db, deployment_id)
1727 if not deployment:
1728 raise HTTPException(
1729 status_code=status.HTTP_404_NOT_FOUND,
1730 detail="Deployment not found",
1731 )
1732 # Inspect-only view, gated through capabilities so course-teachers
1733 # can download the same wizard-uploaded files they can already see
1734 # referenced in the inspect view (logs / detail). Owners + admins
1735 # keep their access. The list/detail strip-pass already hid the
1736 # base64 payload from plain members, so this endpoint stays
1737 # restricted to the owner-view set.
1738 ensure_view_deployment_owner(current_user, deployment, db)
1740 if not deployment.userInputVar:
1741 raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No files")
1742 try:
1743 user_input = json.loads(deployment.userInputVar)
1744 except json.JSONDecodeError:
1745 raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="No files")
1747 tf_block = user_input.get("terraform") if isinstance(user_input, dict) else None
1748 var_value = (tf_block or {}).get(var_name)
1749 if not _looks_like_file_var(var_value):
1750 raise HTTPException(
1751 status_code=status.HTTP_404_NOT_FOUND,
1752 detail=f"No uploaded file under variable '{var_name}'",
1753 )
1754 # ``var_value`` is ``{slot_key: {name, content_b64, size, content_type}}``;
1755 # the slot-level entry IS the file metadata, no extra wrapper.
1756 entry = var_value.get(slot_key)
1757 if not isinstance(entry, dict) or "content_b64" not in entry:
1758 raise HTTPException(
1759 status_code=status.HTTP_404_NOT_FOUND,
1760 detail=f"No file in slot '{slot_key}'",
1761 )
1763 try:
1764 payload = base64.b64decode(entry["content_b64"], validate=True)
1765 except (binascii.Error, ValueError):
1766 # Persisted bytes are corrupt — surface as 500 because there's
1767 # nothing the caller can do; this is a server-side data bug.
1768 raise HTTPException(
1769 status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
1770 detail="Stored file payload is not valid base64",
1771 )
1773 filename = str(entry.get("name") or f"{var_name}-{slot_key}")
1774 content_type = str(entry.get("content_type") or "application/octet-stream")
1775 return Response(
1776 content=payload,
1777 media_type=content_type,
1778 headers={
1779 # ``filename*`` is the RFC 5987 form for non-ASCII names;
1780 # we always emit it alongside the legacy ``filename`` so
1781 # clients without UTF-8 support still see something.
1782 "Content-Disposition": (
1783 f'attachment; filename="{filename}"; '
1784 f"filename*=UTF-8''{filename}"
1785 ),
1786 "Content-Length": str(len(payload)),
1787 },
1788 )
1791# ----------------------------------------------------------------
1792# GET OWN ACCESS CREDENTIALS (member self-service)
1793# ----------------------------------------------------------------
1794@router.get("/{deployment_id}/my-access", response_model=MyAccessResponse)
1795def get_my_access(
1796 deployment_id: UUID,
1797 db: Session = Depends(get_db),
1798 current_user: User = Depends(get_current_user_keycloak),
1799):
1800 """Return the calling member's OWN access credentials for a deployment.
1802 The full terraform ``outputs`` are owner-view-only (they carry every
1803 teammate's credentials in one object). This endpoint is the member
1804 counterpart: a team member — typically a student — retrieves only
1805 THEIR OWN account, extracted server-side from the latest successful
1806 deploy. Teammates' credentials are never included in the response.
1808 Authorisation reuses :func:`ensure_resend_access` with the target set
1809 to the caller themself, which collapses to the member-view gate
1810 (owner, staff, team-member, or direct mapping). A caller with no
1811 access to the deployment gets 403; the resend endpoint uses the same
1812 gate for the self-resend button, so the two stay consistent.
1814 Returns 200 with empty maps when there's no successful deploy yet or
1815 the app issued no per-user credential for this user — the UI renders
1816 a "no credentials yet" state rather than treating it as an error.
1817 """
1818 deployment = crud_deployments.get_deployment(db, deployment_id)
1819 if not deployment:
1820 raise HTTPException(
1821 status_code=status.HTTP_404_NOT_FOUND,
1822 detail="Deployment not found",
1823 )
1824 # Self-target → member-view gate. Non-members get 403 here.
1825 ensure_resend_access(current_user, deployment, current_user.userId, db)
1827 access = deployment_notifier.get_user_access(
1828 db, deployment_id, current_user.userId
1829 )
1830 if access is None:
1831 # No successful deploy yet, user not in any team, or no per-user
1832 # credential in the outputs — surface an empty (but valid) payload.
1833 return MyAccessResponse()
1834 return MyAccessResponse(
1835 user_accounts=access.get("user_accounts", {}),
1836 team_vms=access.get("team_vms", {}),
1837 )
1840# ----------------------------------------------------------------
1841# RESEND ACCESS MAIL FOR ONE TEAM MEMBER
1842# ----------------------------------------------------------------
1843@router.post(
1844 "/{deployment_id}/teams/{team_id}/users/{user_id}/resend-access",
1845 status_code=status.HTTP_202_ACCEPTED,
1846)
1847def resend_access_credentials(
1848 deployment_id: UUID,
1849 team_id: UUID,
1850 user_id: UUID,
1851 db: Session = Depends(get_db),
1852 current_user: User = Depends(get_current_user_keycloak),
1853):
1854 """Re-send the per-user access mail for one team member.
1856 Reuses the original notify pipeline — same template, same
1857 credential extraction from the latest successful DEPLOY task's
1858 ``terraform_outputs``. Useful when the user lost their first mail
1859 or the deploy ran before the user's email got fixed.
1861 Access control: caller must have access to the deployment (owner
1862 or teacher/admin). The endpoint is intentionally idempotent —
1863 each call sends one mail to the targeted user. There's no rate
1864 limit at the API level; SMTP and Gmail's per-account quota are
1865 the natural backstops.
1867 Mapping ResendError to HTTP:
1868 * ``deployment_not_found`` → 404
1869 * ``team_not_in_deployment`` / ``user_not_in_team`` → 404
1870 * ``no_successful_deploy`` → 409 (nothing to resend yet)
1871 * ``no_credentials_for_user`` → 409 (template didn't issue
1872 per-user creds, or matcher missed despite the fuzzy logic)
1873 """
1874 deployment = crud_deployments.get_deployment(db, deployment_id)
1875 if not deployment:
1876 raise HTTPException(
1877 status_code=status.HTTP_404_NOT_FOUND,
1878 detail="Deployment not found",
1879 )
1880 ensure_deployment_access(deployment, current_user, db)
1882 # Members may only re-send their own access mail. Owner-view
1883 # callers (creator, teacher, admin) can resend for anyone in
1884 # any team. Without this check a student in team A could trigger
1885 # a mail to anyone else's address, which is both privacy-leaky
1886 # and a tiny SMTP-amplification vector.
1887 #
1888 # This 403 is decided BEFORE the SMTP-disabled 503 below: a
1889 # not-authorised caller must not learn the SMTP-state of the
1890 # platform — that would be a (small) information disclosure.
1891 if not is_deployment_owner_view(deployment, current_user) and str(user_id) != str(current_user.userId):
1892 raise HTTPException(
1893 status_code=status.HTTP_403_FORBIDDEN,
1894 detail="Members may only resend their own access mail",
1895 )
1897 # SMTP kill-switch check: refuse cleanly with 503 BEFORE running the
1898 # notifier so the response carries the correct semantic ("we chose
1899 # not to send" — operator configuration) rather than the existing
1900 # 502 ("we tried and SMTP rejected" — infrastructure failure).
1901 # Frontend distinguishes the two reasons in the toast text so the
1902 # user understands whether to ask an admin to enable mail or to
1903 # simply retry. The 503 + Retry-After header signals "service
1904 # temporarily unavailable; come back later" semantics.
1905 #
1906 # Order vs. 409 in-flight: a 503 here is non-recoverable until an
1907 # operator flips the env-flag, whereas 409 is a transient state
1908 # the caller can simply wait out. Returning the 503 first matches
1909 # the "service-level concern beats per-request concern" pattern
1910 # used by every other 5xx vs 4xx ordering in this router.
1911 if not email_service.is_smtp_enabled():
1912 raise HTTPException(
1913 status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
1914 detail={"reason": "smtp_disabled"},
1915 headers={"Retry-After": "3600"},
1916 )
1918 # Refuse while another lifecycle action is in flight. Resending
1919 # the access mail relies on the latest successful DEPLOY task's
1920 # ``terraform_outputs``; during pending/running/destroying/
1921 # pausing/resuming the deployment is in transition and the
1922 # credentials might no longer be reachable on the VM (paused →
1923 # SHUTOFF, destroying → tearing down). Returning 409 here keeps
1924 # the user's mental model consistent with the rest of the
1925 # lifecycle gates.
1926 #
1927 # Per-deployment advisory lock is acquired BEFORE the status
1928 # read so a concurrent /pause / /resume / DELETE can't slip a
1929 # transition past us between the check and the mail send. The
1930 # lock is the same one those endpoints take, so the four
1931 # mutators serialise against each other on the same deployment.
1932 crud_locks.acquire_deployment_xact_lock(db, deployment_id)
1933 current_status = crud_deployments.get_deployment_status(db, deployment_id)
1934 if current_status in lifecycle_service.IN_FLIGHT_STATUSES:
1935 raise HTTPException(
1936 status_code=status.HTTP_409_CONFLICT,
1937 detail=(
1938 f"Cannot resend access mail while deployment is '{current_status}'. "
1939 "Wait for the active task to finish."
1940 ),
1941 )
1943 try:
1944 sent = deployment_notifier.resend_user_access(
1945 db, deployment_id, team_id, user_id,
1946 )
1947 except deployment_notifier.ResendError as e:
1948 reason = str(e)
1949 # 404 for "this user/team isn't in this deployment", 409 for
1950 # "the deployment hasn't reached the state where it could
1951 # have emitted credentials yet".
1952 if reason in ("deployment_not_found", "team_not_in_deployment", "user_not_in_team"):
1953 http_status = status.HTTP_404_NOT_FOUND
1954 else:
1955 http_status = status.HTTP_409_CONFLICT
1956 raise HTTPException(status_code=http_status, detail={"reason": reason})
1958 if not sent:
1959 # Template render + payload were fine, only SMTP rejected.
1960 # 502 makes the "upstream service failed" semantics clear so
1961 # the frontend can surface a retry rather than a 4xx that
1962 # implies the request itself was wrong.
1963 raise HTTPException(
1964 status_code=status.HTTP_502_BAD_GATEWAY,
1965 detail={"reason": "smtp_send_failed"},
1966 )
1967 return {"status": "sent"}
1970# ----------------------------------------------------------------
1971# LIVE STREAM — Server-Sent Events for progress + log tail
1972# ----------------------------------------------------------------
1973#
1974# Several kinds of events flow through the stream:
1975#
1976# * ``event: snapshot`` — fired once at connect with the latest task's
1977# current_phase / progress_pct / status. Lets a freshly-loaded page
1978# render the bar at the right position before the worker emits its
1979# next progress update.
1980# * ``event: progress`` — every ``task-progress`` from the worker. The
1981# payload includes ``phase``, ``phase_index``, ``total_phases``,
1982# ``progress_pct``, ``message``.
1983# * ``event: log`` — every ``task-log`` from the worker. The payload
1984# is the LogEntry dict (timestamp, level, category, message, plus
1985# tool/streaming flags for streaming subprocess lines).
1986# * ``event: overflow`` — emitted by the in-process pubsub when a
1987# slow consumer overran its bounded queue.
1988# * comment lines starting with ``:`` are SSE keepalive pings.
1989#
1990# The stream stays open until the deployment reaches a terminal state
1991# (success/failed/cancelled), the client disconnects, or the backend
1992# shuts down. There's no client-driven close — EventSource handles
1993# reconnect automatically.
1994@router.get("/{deployment_id}/stream")
1995async def stream_deployment_events(
1996 deployment_id: UUID,
1997 request: Request,
1998 db: Session = Depends(get_db),
1999 current_user: User = Depends(get_current_user_keycloak),
2000):
2001 """Live progress + log stream for one deployment as Server-Sent Events.
2003 The connection is authenticated with the same Keycloak dependency
2004 used elsewhere; the standard auth middleware also vets the token
2005 before this handler runs. After auth we attach to the in-process
2006 pubsub for this deployment and forward every event to the client.
2007 """
2008 deployment = crud_deployments.get_deployment(db, deployment_id)
2009 if not deployment:
2010 raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Deployment not found")
2011 # Inspect-only view via capabilities. The live stream surfaces
2012 # task-log lines (raw worker stdout incl. terraform output, packer
2013 # build chatter, etc.); course-teachers of the deployment-owner's
2014 # course are in the inspect set, owners and admins keep their access,
2015 # and plain members still see metadata only.
2016 ensure_view_deployment_owner(current_user, deployment, db)
2018 # Snapshot the latest task once before subscribing so the client
2019 # gets a meaningful initial state. Reading happens before the
2020 # generator yields its first chunk to avoid the "subscribed but
2021 # nothing buffered yet" gap.
2022 latest_task = crud_deployments.get_latest_task(db, deployment_id)
2023 snapshot_payload = {
2024 "task_id": str(latest_task.taskId) if latest_task else None,
2025 "status": latest_task.status.value if latest_task else None,
2026 "current_phase": getattr(latest_task, "current_phase", None),
2027 "progress_pct": getattr(latest_task, "progress_pct", None),
2028 "type": latest_task.type.value if latest_task else None,
2029 }
2030 initial_status = latest_task.status if latest_task else None
2032 deployment_id_str = str(deployment_id)
2034 async def event_stream() -> AsyncIterator[bytes]:
2035 queue = pubsub.subscribe(deployment_id_str)
2036 try:
2037 yield _sse_frame("snapshot", snapshot_payload)
2039 # Backfill what's been happening lately. The pubsub keeps a
2040 # bounded ring buffer of recent events per deployment so a
2041 # client connecting mid-stream sees the last few minutes of
2042 # progress / log output instead of an empty tail until the
2043 # next worker line lands. Replays the buffer in order so
2044 # ``streamCurrentPhaseIndex``/``streamProgress`` end up at
2045 # their latest values before the live loop starts.
2046 for past_event in pubsub.recent(deployment_id_str):
2047 event_name = _event_name_for(past_event.get("type"))
2048 yield _sse_frame(event_name, past_event)
2050 # If the task is already in a terminal state we still yield
2051 # the snapshot but close the stream right away — no live
2052 # events will ever arrive for this deployment.
2053 if initial_status in (TaskStatus.SUCCESS, TaskStatus.FAILED, TaskStatus.CANCELLED):
2054 return
2056 # Heartbeat / event-pump loop. Wait up to 15s for an event;
2057 # if nothing arrives, send a ``: keepalive`` comment so
2058 # proxies and the EventSource client don't time out.
2059 while True:
2060 if await request.is_disconnected():
2061 return
2062 try:
2063 event = await asyncio.wait_for(queue.get(), timeout=15.0)
2064 except TimeoutError:
2065 yield b": keepalive\n\n"
2066 continue
2068 event_name = _event_name_for(event.get("type"))
2069 yield _sse_frame(event_name, event)
2071 # Stop streaming once the parent task reaches a
2072 # terminal state. The lifecycle events (succeeded /
2073 # failed / revoked) flow through the same pubsub key,
2074 # so we look for them right here. Without this break
2075 # the connection would dangle until the client closes
2076 # it.
2077 if event.get("type") in ("task-succeeded", "task-failed", "task-revoked"):
2078 return
2079 except asyncio.CancelledError:
2080 # FastAPI cancels the generator on client disconnect.
2081 raise
2082 finally:
2083 pubsub.unsubscribe(deployment_id_str, queue)
2085 return StreamingResponse(
2086 event_stream(),
2087 media_type="text/event-stream",
2088 headers={
2089 "Cache-Control": "no-cache, no-transform",
2090 "X-Accel-Buffering": "no", # disable nginx response buffering
2091 "Connection": "keep-alive",
2092 },
2093 )
2096_EVENT_NAME_MAP: dict[str, str] = {
2097 "task-progress": "progress",
2098 "task-log": "log",
2099 "task-overflow": "overflow",
2100 "task-started": "started",
2101 "task-succeeded": "succeeded",
2102 "task-failed": "failed",
2103 "task-revoked": "revoked",
2104}
2107def _event_name_for(celery_event_type: str | None) -> str:
2108 """Map Celery event type names onto short SSE event names.
2110 Frontend code attaches listeners by these short names rather than
2111 the verbose celery-internal ones; ``_EVENT_NAME_MAP`` is the
2112 single source of truth on both sides of the wire.
2113 """
2114 return _EVENT_NAME_MAP.get(celery_event_type or "", "message")
2117def _sse_frame(event_name: str, payload: dict) -> bytes:
2118 """Serialise one SSE frame.
2120 SSE format:
2122 ```
2123 event: <name>\\n
2124 data: <json>\\n
2125 \\n
2126 ```
2128 Embedded newlines in the JSON would split the frame into multiple
2129 ``data:`` lines per the SSE spec; we use ``json.dumps`` defaults
2130 which keep everything on one line.
2131 """
2132 body = json.dumps(payload, default=str)
2133 return f"event: {event_name}\ndata: {body}\n\n".encode()