Cloud Storage Cost Optimization: S3, Blob, GCS Strategies

# Create a comprehensive lifecycle policy
aws s3api put-bucket-lifecycle-configuration \
  --bucket dodatech-data-lake \
  --lifecycle-configuration '{
    "Rules": [
      {
        "Id": "data-tiering",
        "Status": "Enabled",
        "Filter": {"Prefix": "logs/"},
        "Transitions": [
          {"Days": 30,  "StorageClass": "STANDARD_IA"},
          {"Days": 90,  "StorageClass": "GLACIER_INSTANT_RETRIEVAL"},
          {"Days": 365, "StorageClass": "DEEP_ARCHIVE"}
        ],
        "Expiration": {"Days": 730}
      },
      {
        "Id": "abort-incomplete-multipart",
        "Status": "Enabled",
        "Filter": {"Prefix": ""},
        "AbortIncompleteMultipartUpload": {"DaysAfterInitiation": 7}
      },
      {
        "Id": "expire-old-versions",
        "Status": "Enabled",
        "Filter": {"Prefix": ""},
        "NoncurrentVersionExpiration": {"NoncurrentDays": 90}
      }
    ]
  }'

# Analyze current storage class distribution
aws s3api list-objects-v2 --bucket dodatech-data-lake \
  --query "Contents[].StorageClass" --output text | \
  tr '\t' '\n' | sort | uniq -c | sort -rn

# storage_cost_comparison.py
classes = {
    "S3 Standard":         {"price_per_gb": 0.023},
    "S3 Standard-IA":      {"price_per_gb": 0.0125},
    "S3 One Zone-IA":      {"price_per_gb": 0.010},
    "S3 Glacier Instant":  {"price_per_gb": 0.004},
    "S3 Glacier Flexible": {"price_per_gb": 0.0036},
    "S3 Glacier Deep Arch": {"price_per_gb": 0.00099}
}

data_sizes = {
    "Active data (30 days)": 10_000,
    "Warm data (31-90 days)": 8_000,
    "Cold data (91-365 days)": 15_000,
    "Frozen data (365+ days)": 30_000
}

# Simulate lifecycle costs
monthly_costs = {}
for name, size_gb in data_sizes.items():
    if "Active" in name:
        price = classes["S3 Standard"]["price_per_gb"]
    elif "Warm" in name:
        price = classes["S3 Standard-IA"]["price_per_gb"]
    elif "Cold" in name:
        price = classes["S3 Glacier Instant"]["price_per_gb"]
    else:
        price = classes["S3 Glacier Deep Arch"]["price_per_gb"]
    
    monthly_costs[name] = size_gb * price

total_with = sum(monthly_costs.values())
total_without = sum(data_sizes.values()) * classes["S3 Standard"]["price_per_gb"]

print(f"{'Tier':35} {'Size (GB)':>10} {'Cost/Month':>12}")
print("-" * 60)
for name, size_gb in data_sizes.items():
    print(f"{name:35} {size_gb:>10,} ${monthly_costs[name]:>9.2f}")

print("-" * 60)
print(f"{'Total with lifecycle':35} {'':>10} ${total_with:>9.2f}")
print(f"{'Total without lifecycle (all Standard)':35} {'':>10} ${total_without:>9.2f}")
print(f"{'Savings':35} {'':>10} ${total_without - total_with:>9.2f} ({((total_without - total_with) / total_without) * 100:.0f}%)")

# Set lifecycle management policy on Azure Storage
az storage account management-policy create \
  --account-name dodatechstorage \
  --policy @- <<EOF
{
  "rules": [
    {
      "name": "tier-data",
      "enabled": true,
      "type": "Lifecycle",
      "definition": {
        "filters": {"blobTypes": ["blockBlob"], "prefixMatch": ["logs/"]},
        "actions": {
          "baseBlob": {
            "tierToCool":   {"daysAfterModificationGreaterThan": 30},
            "tierToCold":   {"daysAfterModificationGreaterThan": 90},
            "tierToArchive": {"daysAfterModificationGreaterThan": 365},
            "delete":        {"daysAfterModificationGreaterThan": 730}
          },
          "snapshot": {
            "delete": {"daysAfterCreationGreaterThan": 90}
          }
        }
      }
    }
  ]
}
EOF

# Move a blob to Cool tier manually
az storage blob set-tier \
  --account-name dodatechstorage \
  --container-name logs \
  --name access.log.2026-05-01 \
  --tier Cool

# Check blob tier
az storage blob show \
  --account-name dodatechstorage \
  --container-name logs \
  --name access.log.2026-05-01 \
  --query "properties.blobTier"

# Apply lifecycle policy to GCS bucket
gcloud storage buckets update gs://dodatech-backup \
  --lifecycle-file=- <<EOF
{
  "lifecycle": {
    "rule": [
      {
        "action": {"type": "SetStorageClass", "storageClass": "NEARLINE"},
        "condition": {"age": 30, "matchesStorageClass": ["STANDARD"]}
      },
      {
        "action": {"type": "SetStorageClass", "storageClass": "COLDLINE"},
        "condition": {"age": 90, "matchesStorageClass": ["NEARLINE"]}
      },
      {
        "action": {"type": "SetStorageClass", "storageClass": "ARCHIVE"},
        "condition": {"age": 365, "matchesStorageClass": ["COLDLINE"]}
      },
      {
        "action": {"type": "Delete"},
        "condition": {"age": 1095}
      }
    ]
  }
}
EOF

# List objects by storage class
gsutil du -s -c gs://dodatech-backup/**

# Enable Intelligent-Tiering on a bucket
aws s3api put-bucket-lifecycle-configuration \
  --bucket dodatech-analytics \
  --lifecycle-configuration '{
    "Rules": [{
      "Id": "intelligent-tiering",
      "Status": "Enabled",
      "Transitions": [{
        "Days": 0,
        "StorageClass": "INTELLIGENT_TIERING]
      }]
    }]
  }'

# Monitor tier transitions
aws cloudwatch get-metric-statistics \
  --namespace AWS/S3 \
  --metric-name BucketSizeBytes \
  --dimensions Name=BucketName,Value=dodatech-analytics Name=StorageType,Value=IntelligentTiering_ARCHIVE_ACCESS \
  --start-time 2026-05-01T00:00:00Z \
  --end-time 2026-06-01T00:00:00Z \
  --period 86400 \
  --statistics Average