Databricks Query Audit Logs

testTable = 'default.crime_data_delta'
testDb = 'test'

df = spark.table(testTable)
df.limit(1).collect()

filteredDf = df.filter('victim_age > 20')

filteredDf.write.saveAsTable('{}.audit_cell'.format(testDb))
spark.table('{}.audit_cell'.format(testDb)).limit(1).collect()

spark.sql('DROP TABLE IF EXISTS {}.audit_cell'.format(testDb))

{
  "action": "QUERY",
  "actor": {
    "type": "USER_ACTOR",
    "name": "Taylor",
    "id": "taylor@immuta.com",
    "identityProvider": "okta",
    "impersonatedBy": null
  },
  "sessionId": "abc123456589",
  "actionStatus": "SUCCESS",
  "actionStatusReason": null,
  "actorIp": "1.2.3.4",
  "eventTimestamp": "2022-10-13T20:03:41.013Z",
  "id": "abc123",
  "customerId": "abc123",
  "targetType": "DATASOURCE",
  "targets": [{
    "id": "4",
    "name": "Movies",
    "technology": "DATABRICKS"
  }],
  "auditPayload": {
    "type": "QueryAuditPayload",
    "queryId": "81fe4385-1329-444a-b6d9-b26bce5c8dc7",
    "query": "Project [director#778904]\n+- Filter ((YEAR#778903L = 1999) OR (YEAR#778903L = 2000))\n   +- Relation[movie_id#778901L,Title#778902,Year#778903L,Director#778904,Budget_million#778905,Gross_worldwide#778906L] parquet\n",
    "startTime": "2022-10-13T20:03:41.013Z",
    "endTime": null,
    "duration": null,
    "accessControls": {
      "entitlements": {
        "groups": [],
        "attributes": []
      },
      "policySet": [{
        "type": "SUBSCRIPTION",
        "global": false,
        "subscriptionPolicyType": "MANUAL",
        "ruleAppliedForUser": true
      }]
    },
    "technologyContext": {
      "type": "DatabricksContext",
      "clusterId": "1006-194110-8j0shd5d",
      "clusterName": "databricks-cluster-name",
      "workspaceId": "123456789",
      "pathUris": [
        "dbfs:/user/hive/warehouse/your_database.db/movies"
      ],
      "metastoreTables": ["your_database.movies"],
      "queryLanguage": "python",
      "queryText": "query_success = []\nnum_queries_run = 0\nimpersonate_probability = .20\nspark.sql(\"set immuta.impersonate.user=\")\n\ndef make_fail_query(query):\n  try:\n    spark.sql(\"set immuta.impersonate.user=taylor@databricks.com\")\n    spark.sql(query).toPandas()\n  except: \n    pass\n  \nfor index, query in enumerate(new_queries.values):\n  if(num_queries_run % 100 == 0):\n    print(f\"Queries Successfully Ran: {num_queries_run}/2000, out of total queries ran: {index+1}\")\n  to_impersonate = random.randrange(100)\n  if to_impersonate < impersonate_probability * 100:\n    make_fail_query(query)\n    spark.sql(\"set immuta.impersonate.user=\")\n    num_queries_run += 1\n  else:\n    try:\n      spark.sql(query).toPandas()\n      query_success.append((query, True))\n      num_queries_run += 1\n      if num_queries_run == 2000:\n        break\n    except Exception as e:\n      query_success.append((query, False))\n      \n    ",
      "immutaPluginVersion": "2022.3.0-spark-3.1.1"
    }
  },
  "receivedTimestamp": "2022-10-13T20:03:41.044Z"
}