GET /api/activities/csv
Export activity logs directly to CSV format for analysis in Excel, BI tools, or data warehouses.
Endpoint
Authentication
Requires API Key with read access (all keys have read access by default).
This endpoint supports two authentication methods:
- Bearer Token (Recommended):
Authorization: Bearer <api-key>header - Query Parameter:
?token=<api-key>parameter (for BI tools that cannot set custom headers)
Query Parameters
| Parameter | Required | Description |
|---|---|---|
search |
No | Advanced query string (from convertsearch) |
token |
Conditional | API key (required if not using Authorization header) |
If no search parameter is provided, all activities are exported.
Response Format
The endpoint returns a CSV file with the following columns:
| Column | Description |
|---|---|
ID |
Activity UUID |
Timestamp |
ISO 8601 formatted timestamp |
URL |
Request URL |
Hostname |
Request hostname |
Username |
Authenticated username |
User Group |
User's group membership |
Application |
Matched application name |
Detected |
true/false - Sensitive data detected |
Obfuscated |
true/false - Data was masked |
Blocked |
true/false - Request was blocked |
Bypassed |
true/false - Scanning was bypassed |
Data Types Detected |
Comma-separated list of detected data types |
Data Types Obfuscated |
Comma-separated list of obfuscated data types |
Rules |
Comma-separated list of matched rules |
Content Type |
HTTP Content-Type header |
ICAP Mode |
REQMOD, RESPMOD, or API |
Request Size |
Request body size in bytes |
Response Size |
Response body size in bytes |
Client IP |
Client IP address (if available) |
Client Device |
Device type (if available) |
Examples
Export Last 7 Days
Build a query and export activities from the last 7 days to CSV.
# Build query
QUERY=$(curl -s -X POST "https://your-shield-host:8080/api/activities/convertsearch" \
-H "Authorization: Bearer $API_KEY" \
-H "Content-Type: application/json" \
-d '{
"simpleToAdvanced": {
"timestamp": {"withinLast": {"days": 7, "hours": 0, "minutes": 0}}
}
}' | jq -r '.simpleToAdvanced')
# Export to CSV
curl -X GET "https://your-shield-host:8080/api/activities/csv?search=$(echo $QUERY | jq -sRr @uri)" \
-H "Authorization: Bearer $API_KEY" \
-o activities_last_7_days.csv
import requests
BASE_URL = "https://your-shield-host:8080"
API_KEY = "YOUR_API_KEY"
HEADERS = {"Authorization": f"Bearer {API_KEY}"}
# Build query
query_request = {
"simpleToAdvanced": {
"timestamp": {"withinLast": {"days": 7, "hours": 0, "minutes": 0}}
}
}
query = requests.post(
f"{BASE_URL}/api/activities/convertsearch",
headers=HEADERS,
json=query_request
).json()["simpleToAdvanced"]
# Export to CSV
response = requests.get(
f"{BASE_URL}/api/activities/csv",
headers=HEADERS,
params={"search": query}
)
with open("activities_last_7_days.csv", "wb") as f:
f.write(response.content)
print("Exported to activities_last_7_days.csv")
const axios = require('axios');
const fs = require('fs');
const BASE_URL = 'https://your-shield-host:8080';
const API_KEY = 'YOUR_API_KEY';
const HEADERS = { 'Authorization': `Bearer ${API_KEY}` };
async function exportLast7Days() {
// Build query
const queryRequest = {
simpleToAdvanced: {
timestamp: { withinLast: { days: 7, hours: 0, minutes: 0 } }
}
};
const queryResp = await axios.post(
`${BASE_URL}/api/activities/convertsearch`,
queryRequest,
{ headers: HEADERS }
);
const query = queryResp.data.simpleToAdvanced;
// Export to CSV
const csvResp = await axios.get(
`${BASE_URL}/api/activities/csv`,
{
headers: HEADERS,
params: { search: query },
responseType: 'arraybuffer'
}
);
fs.writeFileSync('activities_last_7_days.csv', csvResp.data);
console.log('Exported to activities_last_7_days.csv');
}
exportLast7Days();
Export Detections with Filters
Export activities from the last 30 days where sensitive data was detected.
# Build query for detected data
QUERY=$(curl -s -X POST "https://your-shield-host:8080/api/activities/convertsearch" \
-H "Authorization: Bearer $API_KEY" \
-H "Content-Type: application/json" \
-d '{
"simpleToAdvanced": {
"detected": ["true"],
"timestamp": {"withinLast": {"days": 30, "hours": 0, "minutes": 0}}
}
}' | jq -r '.simpleToAdvanced')
# Export to CSV
curl -X GET "https://your-shield-host:8080/api/activities/csv?search=$(echo $QUERY | jq -sRr @uri)" \
-H "Authorization: Bearer $API_KEY" \
-o detections_last_30_days.csv
import requests
BASE_URL = "https://your-shield-host:8080"
API_KEY = "YOUR_API_KEY"
HEADERS = {"Authorization": f"Bearer {API_KEY}"}
# Build query
query_request = {
"simpleToAdvanced": {
"detected": ["true"],
"timestamp": {"withinLast": {"days": 30, "hours": 0, "minutes": 0}}
}
}
query = requests.post(
f"{BASE_URL}/api/activities/convertsearch",
headers=HEADERS,
json=query_request
).json()["simpleToAdvanced"]
# Export to CSV
response = requests.get(
f"{BASE_URL}/api/activities/csv",
headers=HEADERS,
params={"search": query}
)
response.raise_for_status()
with open("detections_last_30_days.csv", "wb") as f:
f.write(response.content)
print("Exported to detections_last_30_days.csv")
const axios = require('axios');
const fs = require('fs');
const BASE_URL = 'https://your-shield-host:8080';
const API_KEY = 'YOUR_API_KEY';
const HEADERS = { 'Authorization': `Bearer ${API_KEY}` };
async function exportDetections() {
// Build query
const queryRequest = {
simpleToAdvanced: {
detected: ['true'],
timestamp: { withinLast: { days: 30, hours: 0, minutes: 0 } }
}
};
const queryResp = await axios.post(
`${BASE_URL}/api/activities/convertsearch`,
queryRequest,
{ headers: HEADERS }
);
const query = queryResp.data.simpleToAdvanced;
// Export to CSV
const csvResp = await axios.get(
`${BASE_URL}/api/activities/csv`,
{
headers: HEADERS,
params: { search: query },
responseType: 'arraybuffer'
}
);
fs.writeFileSync('detections_last_30_days.csv', csvResp.data);
console.log('Exported to detections_last_30_days.csv');
}
exportDetections();
Generate CSV URL for BI Tools
Generate a CSV download URL with token authentication for use in BI tools like Tableau or Power BI.
# Build query
QUERY=$(curl -s -X POST "https://your-shield-host:8080/api/activities/convertsearch" \
-H "Authorization: Bearer $API_KEY" \
-H "Content-Type: application/json" \
-d '{
"simpleToAdvanced": {
"timestamp": {"withinLast": {"days": 7, "hours": 0, "minutes": 0}}
}
}' | jq -r '.simpleToAdvanced')
# Build URL with token parameter
ENCODED_QUERY=$(echo $QUERY | jq -sRr @uri)
echo "CSV URL: https://your-shield-host:8080/api/activities/csv?token=$API_KEY&search=$ENCODED_QUERY"
import requests
import urllib.parse
BASE_URL = "https://your-shield-host:8080"
API_KEY = "YOUR_API_KEY"
HEADERS = {"Authorization": f"Bearer {API_KEY}"}
# Build query
query_request = {
"simpleToAdvanced": {
"timestamp": {"withinLast": {"days": 7, "hours": 0, "minutes": 0}}
}
}
query = requests.post(
f"{BASE_URL}/api/activities/convertsearch",
headers=HEADERS,
json=query_request
).json()["simpleToAdvanced"]
# Build URL with token parameter
params = urllib.parse.urlencode({
"token": API_KEY,
"search": query
})
csv_url = f"{BASE_URL}/api/activities/csv?{params}"
print(f"CSV URL: {csv_url}")
# Use this URL in Tableau, Power BI, etc.
const axios = require('axios');
const BASE_URL = 'https://your-shield-host:8080';
const API_KEY = 'YOUR_API_KEY';
const HEADERS = { 'Authorization': `Bearer ${API_KEY}` };
async function generateCSVUrl() {
// Build query
const queryRequest = {
simpleToAdvanced: {
timestamp: { withinLast: { days: 7, hours: 0, minutes: 0 } }
}
};
const queryResp = await axios.post(
`${BASE_URL}/api/activities/convertsearch`,
queryRequest,
{ headers: HEADERS }
);
const query = queryResp.data.simpleToAdvanced;
// Build URL with token parameter
const params = new URLSearchParams({
token: API_KEY,
search: query
});
const csvUrl = `${BASE_URL}/api/activities/csv?${params}`;
console.log(`CSV URL: ${csvUrl}`);
// Use this URL in Tableau, Power BI, etc.
}
generateCSVUrl();
Scheduled Daily Export
Export yesterday's activities automatically (run daily via cron or scheduled task).
#!/bin/bash
# export_yesterday.sh - Export previous day's Shield activities
API_KEY="YOUR_API_KEY"
BASE_URL="https://your-shield-host:8080"
# Calculate yesterday's date
YESTERDAY=$(date -d "yesterday" +%Y-%m-%d)
TODAY=$(date +%Y-%m-%d)
# Build query
QUERY=$(curl -s -X POST "$BASE_URL/api/activities/convertsearch" \
-H "Authorization: Bearer $API_KEY" \
-H "Content-Type: application/json" \
-d "{
\"simpleToAdvanced\": {
\"timestamp\": {
\"inTheRange\": {
\"start\": \"${YESTERDAY}T00:00:00Z\",
\"end\": \"${TODAY}T00:00:00Z\"
}
}
}
}" | jq -r '.simpleToAdvanced')
# Export to CSV with date in filename
FILENAME="shield_activities_$(date -d 'yesterday' +%Y%m%d).csv"
curl -X GET "$BASE_URL/api/activities/csv?search=$(echo $QUERY | jq -sRr @uri)" \
-H "Authorization: Bearer $API_KEY" \
-o "$FILENAME"
echo "Exported $FILENAME"
# Add to crontab: 0 1 * * * /path/to/export_yesterday.sh
#!/usr/bin/env python3
# export_yesterday.py - Export previous day's Shield activities
import requests
from datetime import datetime, timedelta
BASE_URL = "https://your-shield-host:8080"
API_KEY = "YOUR_API_KEY"
HEADERS = {"Authorization": f"Bearer {API_KEY}"}
def export_yesterday():
# Calculate yesterday's date range
today = datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
yesterday = today - timedelta(days=1)
# Build query
query_request = {
"simpleToAdvanced": {
"timestamp": {
"inTheRange": {
"start": yesterday.isoformat() + "Z",
"end": today.isoformat() + "Z"
}
}
}
}
query = requests.post(
f"{BASE_URL}/api/activities/convertsearch",
headers=HEADERS,
json=query_request
).json()["simpleToAdvanced"]
# Export to CSV with date in filename
response = requests.get(
f"{BASE_URL}/api/activities/csv",
headers=HEADERS,
params={"search": query}
)
response.raise_for_status()
filename = f"shield_activities_{yesterday.strftime('%Y%m%d')}.csv"
with open(filename, 'wb') as f:
f.write(response.content)
print(f"Exported {filename}")
if __name__ == "__main__":
export_yesterday()
# Add to crontab: 0 1 * * * /usr/bin/python3 /path/to/export_yesterday.py
#!/usr/bin/env node
// export_yesterday.js - Export previous day's Shield activities
const axios = require('axios');
const fs = require('fs');
const BASE_URL = 'https://your-shield-host:8080';
const API_KEY = 'YOUR_API_KEY';
const HEADERS = { 'Authorization': `Bearer ${API_KEY}` };
async function exportYesterday() {
// Calculate yesterday's date range
const today = new Date();
today.setHours(0, 0, 0, 0);
const yesterday = new Date(today);
yesterday.setDate(yesterday.getDate() - 1);
// Build query
const queryRequest = {
simpleToAdvanced: {
timestamp: {
inTheRange: {
start: yesterday.toISOString(),
end: today.toISOString()
}
}
}
};
const queryResp = await axios.post(
`${BASE_URL}/api/activities/convertsearch`,
queryRequest,
{ headers: HEADERS }
);
const query = queryResp.data.simpleToAdvanced;
// Export to CSV with date in filename
const csvResp = await axios.get(
`${BASE_URL}/api/activities/csv`,
{
headers: HEADERS,
params: { search: query },
responseType: 'arraybuffer'
}
);
const filename = `shield_activities_${yesterday.toISOString().slice(0, 10).replace(/-/g, '')}.csv`;
fs.writeFileSync(filename, csvResp.data);
console.log(`Exported ${filename}`);
}
exportYesterday();
// Add to crontab: 0 1 * * * /usr/bin/node /path/to/export_yesterday.js
Use Cases
Excel Analysis
Download CSV directly for analysis in Microsoft Excel:
curl -X GET "https://your-shield-host:8080/api/activities/csv?token=YOUR_API_KEY" \
-o shield_activities.csv
Open shield_activities.csv in Excel and use pivot tables, charts, and filters for analysis.
BI Tool Integration
Use the CSV URL with query parameter authentication in tools like:
- Tableau - Add as Web Data Connector
- Power BI - Import from Web
- Google Data Studio - CSV file connector
- Looker - External data source
Data Warehouse Loading
Schedule regular CSV exports and load into data warehouses:
Export and Upload to S3
# Export and upload to S3 using AWS CLI
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
FILENAME="shield/activities_${TIMESTAMP}.csv"
# Export CSV from Shield
curl -X GET "https://your-shield-host:8080/api/activities/csv?search=$QUERY" \
-H "Authorization: Bearer $API_KEY" \
-o "/tmp/activities.csv"
# Upload to S3
aws s3 cp /tmp/activities.csv "s3://analytics-data/${FILENAME}"
echo "Uploaded to s3://analytics-data/${FILENAME}"
# Export and upload to S3
import boto3
response = requests.get(
f"{BASE_URL}/api/activities/csv",
headers=HEADERS,
params={"search": query}
)
s3 = boto3.client('s3')
s3.put_object(
Bucket='analytics-data',
Key=f'shield/activities_{datetime.now().strftime("%Y%m%d_%H%M%S")}.csv',
Body=response.content
)
const AWS = require('aws-sdk');
const axios = require('axios');
// Export and upload to S3
const response = await axios.get(
`${BASE_URL}/api/activities/csv`,
{
headers: HEADERS,
params: { search: query },
responseType: 'arraybuffer'
}
);
const s3 = new AWS.S3();
const timestamp = new Date().toISOString().replace(/[-:]/g, '').slice(0, 15);
await s3.putObject({
Bucket: 'analytics-data',
Key: `shield/activities_${timestamp}.csv`,
Body: response.data
}).promise();
console.log(`Uploaded to s3://analytics-data/shield/activities_${timestamp}.csv`);
Compliance Reporting
Generate monthly compliance reports:
Monthly PII Compliance Report
#!/bin/bash
# Generate monthly PII compliance report
YEAR=2024
MONTH=1
# Get PII data type IDs
DATATYPES=$(curl -s "https://your-shield-host:8080/api/datatypes" \
-H "Authorization: Bearer $API_KEY")
SSN_ID=$(echo "$DATATYPES" | jq -r '.items[] | select(.type=="US_SSN") | .id')
CC_ID=$(echo "$DATATYPES" | jq -r '.items[] | select(.type=="CREDIT_CARD") | .id')
EMAIL_ID=$(echo "$DATATYPES" | jq -r '.items[] | select(.type=="EMAIL_ADDRESS") | .id')
PHONE_ID=$(echo "$DATATYPES" | jq -r '.items[] | select(.type=="PHONE_NUMBER") | .id')
# Calculate month range
START=$(date -d "${YEAR}-${MONTH}-01" +%Y-%m-%dT00:00:00Z)
END=$(date -d "${YEAR}-${MONTH}-01 +1 month" +%Y-%m-%dT00:00:00Z)
# Build query
QUERY=$(curl -s -X POST "https://your-shield-host:8080/api/activities/convertsearch" \
-H "Authorization: Bearer $API_KEY" \
-H "Content-Type: application/json" \
-d "{
\"simpleToAdvanced\": {
\"timestamp\": {
\"inTheRange\": {
\"start\": \"$START\",
\"end\": \"$END\"
}
},
\"detectedDatatypes\": [\"$SSN_ID\", \"$CC_ID\", \"$EMAIL_ID\", \"$PHONE_ID\"]
}
}" | jq -r '.simpleToAdvanced')
# Export CSV
FILENAME="pii_compliance_report_${YEAR}_$(printf '%02d' $MONTH).csv"
curl -X GET "https://your-shield-host:8080/api/activities/csv?search=$(echo $QUERY | jq -sRr @uri)" \
-H "Authorization: Bearer $API_KEY" \
-o "$FILENAME"
echo "Compliance report: $FILENAME"
def generate_compliance_report(year, month):
"""Generate monthly compliance report CSV."""
# Get PII data type IDs
datatypes = requests.get(f"{BASE_URL}/api/datatypes", headers=HEADERS).json()
pii_ids = [dt["id"] for dt in datatypes["items"]
if dt["type"] in ["US_SSN", "CREDIT_CARD", "EMAIL_ADDRESS", "PHONE_NUMBER"]]
# Calculate month range
start = datetime(year, month, 1)
end = datetime(year, month + 1, 1) if month < 12 else datetime(year + 1, 1, 1)
# Build query
query_request = {
"simpleToAdvanced": {
"timestamp": {
"inTheRange": {
"start": start.isoformat() + "Z",
"end": end.isoformat() + "Z"
}
},
"detectedDatatypes": pii_ids
}
}
query = requests.post(
f"{BASE_URL}/api/activities/convertsearch",
headers=HEADERS,
json=query_request
).json()["simpleToAdvanced"]
# Export CSV
response = requests.get(
f"{BASE_URL}/api/activities/csv",
headers=HEADERS,
params={"search": query}
)
filename = f"pii_compliance_report_{year}_{month:02d}.csv"
with open(filename, 'wb') as f:
f.write(response.content)
print(f"Compliance report: {filename}")
generate_compliance_report(2024, 1)
async function generateComplianceReport(year, month) {
// Get PII data type IDs
const datatypesResp = await axios.get(
`${BASE_URL}/api/datatypes`,
{ headers: HEADERS }
);
const piiTypes = ['US_SSN', 'CREDIT_CARD', 'EMAIL_ADDRESS', 'PHONE_NUMBER'];
const piiIds = datatypesResp.data.items
.filter(dt => piiTypes.includes(dt.type))
.map(dt => dt.id);
// Calculate month range
const start = new Date(year, month - 1, 1);
const end = new Date(year, month, 1);
// Build query
const queryRequest = {
simpleToAdvanced: {
timestamp: {
inTheRange: {
start: start.toISOString(),
end: end.toISOString()
}
},
detectedDatatypes: piiIds
}
};
const queryResp = await axios.post(
`${BASE_URL}/api/activities/convertsearch`,
queryRequest,
{ headers: HEADERS }
);
const query = queryResp.data.simpleToAdvanced;
// Export CSV
const csvResp = await axios.get(
`${BASE_URL}/api/activities/csv`,
{
headers: HEADERS,
params: { search: query },
responseType: 'arraybuffer'
}
);
const filename = `pii_compliance_report_${year}_${month.toString().padStart(2, '0')}.csv`;
fs.writeFileSync(filename, csvResp.data);
console.log(`Compliance report: ${filename}`);
}
generateComplianceReport(2024, 1);
Performance Considerations
Large Exports
For queries returning millions of records:
- Use time-based batching - Export data in smaller time windows (daily instead of monthly)
- Export during off-peak hours - Schedule large exports when Shield is less busy
- Stream processing - Process CSV data in chunks rather than loading entirely into memory
Stream Large CSV Export
const axios = require('axios');
const fs = require('fs');
// Stream large CSV export
const response = await axios.get(
`${BASE_URL}/api/activities/csv`,
{
headers: HEADERS,
params: { search: query },
responseType: 'stream'
}
);
const writer = fs.createWriteStream('large_export.csv');
response.data.pipe(writer);
await new Promise((resolve, reject) => {
writer.on('finish', resolve);
writer.on('error', reject);
});
console.log('Export completed: large_export.csv');
Error Responses
| Status Code | Description | Resolution |
|---|---|---|
| 400 | Invalid query syntax | Verify query string with convertsearch |
| 401 | Invalid or missing API key | Check authentication (header or query param) |
| 500 | Server error | Check Shield logs, try smaller time range |
Related Topics
- POST /api/activities/convertsearch - Build query filters
- GET /api/activities - Query activities (JSON format)
- Extracting Activity Data Workflow - Complete workflow guide