S3 Tables

Required settings

Table Bucket ARN
S3 Bucket
Service account

Find table bucket ARN

Creating a S3 bucket

We will need a S3 bucket created so that we can use it to stage our delta files. Below is a script that can create a bucket and attach a lifecycle policy to remove old files.

Terraform script

variable "bucket_name" {
  type = string
}

resource "aws_s3_bucket" "bucket" {
  bucket = var.bucket_name
}

resource "aws_s3_bucket_lifecycle_configuration" "bucket_lifecycle" {
  rule {
    id     = "DeleteOldObjects"
    status = "Enabled"

    expiration {
      days = 1
    }
  }

  bucket = aws_s3_bucket.bucket.id
}

Creating a service account

The service account will need the following permissions:

S3 Tables API access
S3 bucket access (to the delta bucket)

Terraform script

variable "table_bucket_name" {
  type = string
}

variable "bucket_name" {
  type = string
}

variable "service_account_name" {
  type = string
}

resource "aws_iam_role" "s3_tables_role" {
  name = "S3TablesRole"

  assume_role_policy = jsonencode({
    Version   = "2012-10-17",
    Statement = [
      {
        Action = "sts:AssumeRole",
        Principal = {
          Service = "ec2.amazonaws.com"
        },
        Effect = "Allow",  
        Sid    = ""
      }
    ]
  })
}

resource "aws_iam_policy" "s3_tables_policy" {
  name        = "S3TablesPolicy"
  description = "Policy that grants access to S3 tables and related resources"

  policy = jsonencode({
    Version = "2012-10-17",
    Statement = [
      {
        Effect = "Allow",
        Action = [
          "s3tables:CreateTable",
          "s3tables:GetTableBucket",
          "s3tables:ListTables",
          "s3tables:GetNamespace",
          "s3tables:ListNamespace",
          "s3tables:CreateNamespace",
          "s3tables:DeleteNamespace"
        ],
        Resource = [
          "arn:aws:s3tables:*:*:bucket/${var.table_bucket_name}",
          "arn:aws:s3tables:*:*:bucket/${var.table_bucket_name}/*"
        ]
      },
      {
        Effect = "Allow",
        Action = [
          "s3tables:GetTable",
          "s3tables:GetTableData",
          "s3tables:GetTableMetadataLocation",
          "s3tables:GetTableData",
          "s3tables:PutTableData",
          "s3tables:UpdateTableMetadataLocation"
        ],
        Resource = [
          "arn:aws:s3tables:*:*:bucket/${var.table_bucket_name}/table/*"
        ]
      },
      {
        Effect = "Allow",
        Action = [
          "s3:GetObject",
          "s3:PutObject",
          "s3:DeleteObject",
          "s3:ListBucket"
        ],
        Resource = [
          "arn:aws:s3:::${var.bucket_name}/*",
          "arn:aws:s3:::${var.bucket_name}"
        ]
      }
    ]
  })
}

resource "aws_iam_role_policy_attachment" "s3_tables_role_policy_attachment" {
  role       = aws_iam_role.s3_tables_role.name
  policy_arn = aws_iam_policy.s3_tables_policy.arn
}

resource "aws_iam_user" "s3_tables_user" {
  name = var.service_account_name
  path = "/"
}

resource "aws_iam_user_policy_attachment" "s3_tables_user_policy_attachment" {
  user       = aws_iam_user.s3_tables_user.name
  policy_arn = aws_iam_policy.s3_tables_policy.arn
}

resource "aws_iam_access_key" "s3_tables_user_key" {
  user = aws_iam_user.s3_tables_user.name
}

output "aws_access_key_id" {
  value     = aws_iam_access_key.s3_tables_user_key.id
  sensitive = true
}

output "aws_secret_access_key" {
  value     = aws_iam_access_key.s3_tables_user_key.secret
  sensitive = true
}

Home

Concepts

Connectors

Tables

Monitoring

Artie Dashboard

Legal

Required settings

Creating a S3 bucket

Creating a service account

Home

Concepts

Connectors

Tables

Monitoring

Artie Dashboard

Legal

​Required settings

​Creating a S3 bucket

​Creating a service account

Required settings

Creating a S3 bucket

Creating a service account