Hi dear Zindi Barbados Traffic Analysis Challenge Contestants,
When I examined the data section, I saw that the total size of the datasets under the name "normanniles" did not exceed 500 GB. However, there is some information below about a dataset larger than 500 GB. But I cannot find this dataset. How can I access it?
@enes_lt Run following scripts it is present on bucket after setting the paths you want
import os import requests from tqdm import tqdm import xml.etree.ElementTree as ET BUCKET = "https://storage.googleapis.com/brb-traffic-full/" DOWNLOAD_DIR = os.path.expanduser("~/Downloads/brb_videos_full") LOG = os.path.expanduser("~/Downloads/brb_full_done.txt") os.makedirs(DOWNLOAD_DIR, exist_ok=True) done = set() if os.path.exists(LOG): with open(LOG, "r") as f: done = set(f.read().splitlines()) def list_all_files(): marker = "" all_files = [] while True: url = BUCKET + "?marker=" + marker xml = requests.get(url).text root = ET.fromstring(xml) keys = [e.text for e in root.iter() if e.tag.endswith("Key") and e.text.endswith(".mp4")] all_files.extend(keys) if not keys or "false " in xml:
break
marker = keys[-1]
return all_files
# Pre‑compute total size
all_files = list_all_files()
total_size = 0
sizes = {}
for key in all_files:
head = requests.head(BUCKET + key)
size = head.headers.get('content-length')
if size is not None:
size = int(size)
sizes[key] = size
total_size += size
else:
sizes[key] = None
print(f"Total to download: {total_size / (1024*1024):.1f} MB across {len(all_files)} files")
with tqdm(total=total_size, unit='B', unit_scale=True, desc="Total Download") as pbar:
for key in all_files:
name = os.path.basename(key)
if name in done:
continue
filepath = os.path.join(DOWNLOAD_DIR, name)
if os.path.exists(filepath):
with open(LOG, "a") as f:
f.write(name + "\n")
continue
response = requests.get(BUCKET + key, stream=True)
response.raise_for_status()
for chunk in response.iter_content(chunk_size=8192):
if chunk:
with open(filepath, "ab") as f:
f.write(chunk)
pbar.update(len(chunk))
with open(LOG, "a") as f:
f.write(name + "\n")
print("✅ Download complete.")
.
`
.
.
.
.
.
.
.
.
.