Coverage for backend/django/core/auxiliary/services/ml_artifacts.py: 70%

1"""Reference-aware early deletion for autonomously expiring ML artifacts.""" 

2 

3import logging 

4 

5from botocore.exceptions import ClientError 

6 

7from core.auxiliary.models.MLModel import MLModel 

8from core.auxiliary.services.object_storage.s3 import delete_object 

9 

10 

11logger = logging.getLogger(__name__) 

12 

13 

14def ml_artifact_is_referenced( 

15 *, 

16 bucket: str, 

17 key: str, 

18 excluded_model_id: int | None = None, 

19) -> bool: 

20 """Return whether any working or historical ML row references an object.""" 

21 

22 csv_references = MLModel.all_states.filter( 

23 csv_bucket=bucket, 

24 csv_object_key=key, 

25 ) 

26 result_references = MLModel.all_states.filter( 

27 test_results_bucket=bucket, 

28 test_results_key=key, 

29 ) 

30 if excluded_model_id is not None: 30 ↛ 31line 30 didn't jump to line 31 because the condition on line 30 was never true

31 csv_references = csv_references.exclude(id=excluded_model_id) 

32 result_references = result_references.exclude(id=excluded_model_id) 

33 return csv_references.exists() or result_references.exists() 

34 

35 

36def delete_ml_artifact_if_unreferenced( 

37 *, 

38 bucket: str, 

39 key: str, 

40 excluded_model_id: int | None = None, 

41) -> bool: 

42 """Best-effort delete an ML artifact once its final content row is gone. 

43 

44 ``UploadSession`` is an operational audit record rather than an artifact 

45 owner, so only current or historical ``MLModel`` rows defer early deletion. 

46 Object-storage lifecycle rules remain the final autonomous retention bound. 

47 Returns whether the object was absent or deleted after its final retaining 

48 model reference disappeared. Storage failures are logged for operational 

49 retry without failing the committed database operation. 

50 """ 

51 

52 if ml_artifact_is_referenced( 

53 bucket=bucket, 

54 key=key, 

55 excluded_model_id=excluded_model_id, 

56 ): 

57 return False 

58 try: 

59 delete_object(bucket=bucket, key=key) 

60 except ClientError as exc: 

61 error_code = exc.response.get("Error", {}).get("Code") 

62 if error_code not in {"NoSuchBucket", "NoSuchKey", "404"}: 

63 logger.exception( 

64 "Failed to delete unreferenced ML artifact %s/%s.", 

65 bucket, 

66 key, 

67 extra={"bucket": bucket, "object_key": key}, 

68 ) 

69 return False 

70 logger.warning( 

71 "Skipping ML artifact deletion because object storage no longer has %s/%s.", 

72 bucket, 

73 key, 

74 extra={"bucket": bucket, "object_key": key}, 

75 ) 

76 except Exception: 

77 logger.exception( 

78 "Failed to delete unreferenced ML artifact %s/%s.", 

79 bucket, 

80 key, 

81 extra={"bucket": bucket, "object_key": key}, 

82 ) 

83 return False 

84 return True