0

I want to download many files (> tens of millions). I have the URL for each file. I have the list of URLs in a file URLs.txt:

http://mydomain.com/0wd.pdf
http://mydomain.com/asz.pdf
http://mydomain.com/axz.pdf
http://mydomain.com/b00.pdf
http://mydomain.com/bb0.pdf
etc.

I can download them via wget -i URLs.txt, however it would go over the maximum number of files that can be placed in one folder.

How can I download this large list of URLs so that the downloaded files are split into subfolders containing the first letter of the filenames? E.g.,:

0/0wd.pdf
a/asz.pdf
a/axz.pdf
b/b00.pdf
b/bb0.pdf
etc.

If that matters, I use Ubuntu.

3
  • 2
    I would split the URLs.txt file into multiple files (URLs-0.txt, URLs-a.txt, ...) and then loop over them doing wget -i on each in the appropriate subdirectory.
    – NickD
    Dec 21, 2023 at 23:18
  • 1
    Take SHA of the URL. Pick off a couple of prefix nybbles, and use that as the directory name.
    – J_H
    Dec 22, 2023 at 2:47
  • 1
    You might do a frequency count on the output of whatever algorithm you choose. If the results are badly skewed, you might still have a mix of too-large and small directories. I assume the names shown are not real: an alphabet of A-Za-z0-9 of length 3 has less than 250,000 unique names. Dec 22, 2023 at 13:22

2 Answers 2

1

Maybe something like:

awk -F/ '{print substr($NF, 1, 1), $0}' urls.txt |
  xargs -L1 bash -c 'mkdir -p -- "$0" && curl -sSF -O --output-dir "$0" "$1"'

The awk prefixes each line with the first character of the filename, which is then used to select an output directory in the curl command. You could use the -P option to the GNU implementation of xargs to run multiple fetches in parallel.

That assumes URLs don't contain whitespace, quotes or backslashes but URLs should not contain those other than URI-encoded (even if curl is able to cope with them and do the URI-encoding by itself).

Given your sample input, running the above produces:

.
├── 0
│   └── 0wd.pdf
├── a
│   ├── asz.pdf
│   └── axz.pdf
└── b
    ├── b00.pdf
    └── bb0.pdf
-1

ChatGPT gave some working code in Python (I confirm that it works with Python 3.11):

import os import requests

def download_files_with_subfolders(url_file):
    with open(url_file, 'r') as file:
        for url in file:
            url = url.strip()
            filename = os.path.basename(url)
            first_letter = filename[0]

            # Create subfolder if it doesn't exist
            subfolder = os.path.join(first_letter, '')
            os.makedirs(subfolder, exist_ok=True)

            # Download the file
            response = requests.get(url)
            if response.status_code == 200:
                file_path = os.path.join(subfolder, filename)
                with open(file_path, 'wb') as file:
                    file.write(response.content)
                print(f"Downloaded: {url} -> {file_path}")
            else:
                print(f"Failed to download: {url} (Status code: {response.status_code})")

if __name__ == "__main__":
    urls_file = "somefile.txt"
    download_files_with_subfolders(urls_file) 

with somefile.txt containing:

http://mydomain.com/0wd.pdf
http://mydomain.com/asz.pdf
http://mydomain.com/axz.pdf
http://mydomain.com/b00.pdf
http://mydomain.com/bb0.pdf
etc.

More advanced variants:

  1. Keep the last modified date from the response headers (the code is also mostly from ChatGPT):
import requests
import os
from datetime import datetime

def download_file(url, local_filename):
    # Send a GET request to the server
    response = requests.get(url, stream=True)

    # Check if the request was successful (status code 200)
    if response.status_code == 200:
        # Get the last modified date from the response headers
        last_modified_header = response.headers.get('Last-Modified')
        last_modified_date = datetime.strptime(last_modified_header, '%a, %d %b %Y %H:%M:%S %Z')

        # Save the content to a local file while preserving the original date
        with open(local_filename, 'wb') as f:
            for chunk in response.iter_content(chunk_size=128):
                f.write(chunk)

        # Set the local file's last modified date to match the original date
        os.utime(local_filename, (last_modified_date.timestamp(), last_modified_date.timestamp()))

        print(f"Downloaded {local_filename} with the original date {last_modified_date}")
    else:
        print(f"Failed to download file. Status code: {response.status_code}")


def download_files_with_subfolders(url_file):
    with open(url_file, 'r') as file:
        for url in file:
            url = url.strip()
            filename = os.path.basename(url)
            first_letter = filename[0]

            # Create subfolder if it doesn't exist
            subfolder = os.path.join(first_letter, '')
            os.makedirs(subfolder, exist_ok=True)

            file_path = os.path.join(subfolder, filename)
            download_file(url, file_path)

if __name__ == "__main__":
    urls_file = "somefile.txt"
    download_files_with_subfolders(urls_file)
  1. Multithreaded download:
import requests
import os
from datetime import datetime

from multiprocessing.dummy import Pool as ThreadPool

def download_file(url, local_filename):
    # Send a GET request to the server
    response = requests.get(url, stream=True)

    # Check if the request was successful (status code 200)
    if response.status_code == 200:
        # Get the last modified date from the response headers
        last_modified_header = response.headers.get('Last-Modified')
        last_modified_date = datetime.strptime(last_modified_header, '%a, %d %b %Y %H:%M:%S %Z')

        # Save the content to a local file while preserving the original date
        with open(local_filename, 'wb') as f:
            for chunk in response.iter_content(chunk_size=128):
                f.write(chunk)

        # Set the local file's last modified date to match the original date
        os.utime(local_filename, (last_modified_date.timestamp(), last_modified_date.timestamp()))

        print(f"Downloaded {local_filename} with the original date {last_modified_date}")
    else:
        print(f"Failed to download file. Status code: {response.status_code}")


def download_files_with_subfolders(url_file, num_threads=4):
    download_arguments = []
    with open(url_file, 'r') as file:
        for url in file:
            url = url.strip()
            filename = os.path.basename(url)
            first_letter = filename[0]

            # Create subfolder if it doesn't exist
            subfolder = os.path.join(first_letter, '')
            os.makedirs(subfolder, exist_ok=True)

            file_path = os.path.join(subfolder, filename)
            download_arguments.append((url, file_path))

    pool = ThreadPool(num_threads)
    results = pool.starmap(download_file, download_arguments)


if __name__ == "__main__":
    urls_file = "somefile.txt"
    download_files_with_subfolders(urls_file, num_threads=10)
  1. Make one folder for the first letter, and a subfolder for the second letter. E.g.,:
0/w/0wd.pdf
a/s/asz.pdf
a/x/axz.pdf
b/0/b00.pdf
b/b/bb0.pdf
etc.

Code:

import requests
import os
from datetime import datetime

from multiprocessing.dummy import Pool as ThreadPool

def download_file(url, local_filename):
    # Send a GET request to the server
    response = requests.get(url, stream=True)

    # Check if the request was successful (status code 200)
    if response.status_code == 200:
        # Get the last modified date from the response headers
        last_modified_header = response.headers.get('Last-Modified')
        last_modified_date = datetime.strptime(last_modified_header, '%a, %d %b %Y %H:%M:%S %Z')

        # Save the content to a local file while preserving the original date
        with open(local_filename, 'wb') as f:
            for chunk in response.iter_content(chunk_size=128):
                f.write(chunk)

        # Set the local file's last modified date to match the original date
        os.utime(local_filename, (last_modified_date.timestamp(), last_modified_date.timestamp()))

        print(f"Downloaded {local_filename} with the original date {last_modified_date}")
    else:
        print(f"Failed to download file. Status code: {response.status_code}")


def download_files_with_subfolders(url_file, num_threads=4):
    download_arguments = []
    with open(url_file, 'r') as file:
        for url in file:
            url = url.strip()
            filename = os.path.basename(url)
            first_letter = filename[0]
            second_letter = filename[1]

            # Create subfolder if it doesn't exist
            subfolder = os.path.join(first_letter, '')
            os.makedirs(subfolder, exist_ok=True)
            subsubfolder = os.path.join(first_letter, second_letter)
            os.makedirs(subsubfolder, exist_ok=True)

            file_path = os.path.join(subsubfolder, filename)
            download_arguments.append((url, file_path))

    pool = ThreadPool(num_threads)
    results = pool.starmap(download_file, download_arguments)


if __name__ == "__main__":
    urls_file = "somefile.txt"
    download_files_with_subfolders(urls_file, num_threads=10)

You must log in to answer this question.

Not the answer you're looking for? Browse other questions tagged .