Mirrors/Stirling-PDF
Mirrors/Stirling-PDF
1
0
Fork 0
mirror of https://github.com/Frooodle/Stirling-PDF.git synced 2025-11-16 01:21:16 +01:00
Code Issues Packages Projects Releases Wiki Activity
cf560b7488
Stirling-PDF/scripts/download_pdf_collection.py
Anthony Stirling e915e1aa7d tpye3 text edit init
2025-11-09 12:43:33 +00:00

584 lines
28 KiB
Python
Raw Blame History

#!/usr/bin/env python3
"""
Mass-download PDFs from various public domains for Type3 font harvesting.
Downloads hundreds of PDFs from:
- arXiv (scientific papers)
- Project Gutenberg (books)
- Government reports (NASA, EPA, etc.)
- Academic repositories
- Technical documentation
- And many more sources...
Run with: python scripts/download_pdf_collection.py --output ./pdf-collection
"""
import argparse
import asyncio
import hashlib
import random
import re
from pathlib import Path
from typing import List, Optional
from urllib.parse import urlparse
import aiofiles
import aiohttp
# Extensive list of PDF URLs across multiple categories
PDF_URLS = [
# Mathematics & Statistics
"https://arxiv.org/pdf/2103.14030.pdf", # Swin Transformer
"https://arxiv.org/pdf/2010.11929.pdf", # Vision Transformer
"https://arxiv.org/pdf/2005.14165.pdf", # GPT-3 Paper
"https://arxiv.org/pdf/1910.10683.pdf", # T5 Text-to-Text Transformer
"https://arxiv.org/pdf/1810.04805.pdf", # BERT
"https://arxiv.org/pdf/1706.03762.pdf", # Attention Is All You Need
"https://arxiv.org/pdf/1603.04467.pdf", # TensorFlow White Paper
"https://arxiv.org/pdf/1511.06434.pdf", # DCGAN
"https://arxiv.org/pdf/1506.03378.pdf", # LIME
"https://arxiv.org/pdf/1409.1556.pdf", # VGGNet
"https://arxiv.org/pdf/1312.6114.pdf", # Variational Autoencoders
"https://arxiv.org/pdf/1211.4240.pdf", # AlexNet
"https://arxiv.org/pdf/1106.1813.pdf", # CIFAR-10
"https://arxiv.org/pdf/1003.0358.pdf", # SVM Theory
"https://arxiv.org/pdf/0909.4061.pdf", # Random Forests
# Physics
"https://arxiv.org/pdf/2303.08774.pdf", # Quantum Computing
"https://arxiv.org/pdf/2201.04294.pdf", # Dark Matter Research
"https://arxiv.org/pdf/2105.00552.pdf", # Gravitational Waves
"https://arxiv.org/pdf/2004.00007.pdf", # Particle Physics
"https://arxiv.org/pdf/1906.10176.pdf", # Cosmology
"https://arxiv.org/pdf/1807.02101.pdf", # String Theory
"https://arxiv.org/pdf/1708.05671.pdf", # Quantum Entanglement
"https://arxiv.org/pdf/1605.08625.pdf", # Astrophysics
# Computer Science
"https://arxiv.org/pdf/2204.02311.pdf", # PaLM Language Model
"https://arxiv.org/pdf/2112.07804.pdf", # Stable Diffusion
"https://arxiv.org/pdf/2107.03374.pdf", # Codex
"https://arxiv.org/pdf/2010.02559.pdf", # Neural Architecture Search
"https://arxiv.org/pdf/1912.01703.pdf", # YOLOv4
"https://arxiv.org/pdf/1905.11946.pdf", # EfficientNet
"https://arxiv.org/pdf/1812.01187.pdf", # BERT Large
"https://arxiv.org/pdf/1801.00631.pdf", # Transformer Applications
"https://arxiv.org/pdf/1704.04861.pdf", # MobileNet
"https://arxiv.org/pdf/1602.07360.pdf", # SqueezeNet
"https://arxiv.org/pdf/1512.03385.pdf", # ResNet
"https://arxiv.org/pdf/1506.02640.pdf", # YOLO
"https://arxiv.org/pdf/1502.03167.pdf", # Batch Normalization
"https://arxiv.org/pdf/1412.6980.pdf", # Adam Optimizer
"https://arxiv.org/pdf/1409.4842.pdf", # GoogLeNet
"https://arxiv.org/pdf/1312.5602.pdf", # Deep Q-Network
"https://arxiv.org/pdf/1301.3781.pdf", # Word2Vec
"https://arxiv.org/pdf/1207.0580.pdf", # Dropout
"https://arxiv.org/pdf/1102.1803.pdf", # ImageNet Classification
# Government Reports
"https://www.nasa.gov/sites/default/files/atoms/files/2023_nasa_annual_report.pdf",
"https://www.nasa.gov/sites/default/files/atoms/files/2022_nasa_annual_report.pdf",
"https://www.nasa.gov/sites/default/files/atoms/files/2021_nasa_annual_report.pdf",
"https://www.epa.gov/system/files/documents/2023-01/epa-strategic-plan-2022-2026.pdf",
"https://www.epa.gov/system/files/documents/2022-12/epa-annual-report-2022.pdf",
"https://www.nist.gov/system/files/documents/2023/02/15/NIST%20Annual%20Report%202022.pdf",
"https://www.nist.gov/system/files/documents/2022/03/01/NIST%20Annual%20Report%202021.pdf",
"https://www.noaa.gov/sites/default/files/2023-03/NOAA%20Annual%20Report%202022.pdf",
"https://www.fda.gov/media/165773/download",
"https://www.fda.gov/media/159722/download",
"https://www.cdc.gov/mmwr/PDF/wk/mm7201.pdf",
"https://www.cdc.gov/nchs/data/nvsr/nvsr71/nvsr71-01.pdf",
"https://www.bls.gov/opub/mlr/2023/article/pdf/labor-force-projections-2022-2032.pdf",
"https://www.bls.gov/opub/mlr/2023/article/pdf/union-membership-2022.pdf",
"https://www.census.gov/content/dam/Census/library/publications/2023/demo/p60-280.pdf",
"https://www.energy.gov/sites/default/files/2023-04/DOE%20Annual%20Report%202022.pdf",
# Project Gutenberg Classics
"https://www.gutenberg.org/files/1342/1342-pdf.pdf", # Pride and Prejudice
"https://www.gutenberg.org/files/84/84-pdf.pdf", # Frankenstein
"https://www.gutenberg.org/files/11/11-pdf.pdf", # Alice in Wonderland
"https://www.gutenberg.org/files/1661/1661-pdf.pdf", # Sherlock Holmes
"https://www.gutenberg.org/files/98/98-pdf.pdf", # Tale of Two Cities
"https://www.gutenberg.org/files/2701/2701-pdf.pdf", # Moby Dick
"https://www.gutenberg.org/files/2542/2542-pdf.pdf", # A Doll's House
"https://www.gutenberg.org/files/174/174-pdf.pdf", # Picture of Dorian Gray
"https://www.gutenberg.org/files/1952/1952-pdf.pdf", # The Yellow Wallpaper
"https://www.gutenberg.org/files/1080/1080-pdf.pdf", # A Modest Proposal
"https://www.gutenberg.org/files/43/43-pdf.pdf", # Dr. Jekyll and Mr. Hyde
"https://www.gutenberg.org/files/345/345-pdf.pdf", # Dracula
"https://www.gutenberg.org/files/5200/5200-pdf.pdf", # Metamorphosis
"https://www.gutenberg.org/files/76/76-pdf.pdf", # Adventures of Huckleberry Finn
"https://www.gutenberg.org/files/74/74-pdf.pdf", # Tom Sawyer
"https://www.gutenberg.org/files/1260/1260-pdf.pdf", # Jane Eyre
"https://www.gutenberg.org/files/768/768-pdf.pdf", # Wuthering Heights
"https://www.gutenberg.org/files/219/219-pdf.pdf", # Heart of Darkness
"https://www.gutenberg.org/files/1184/1184-pdf.pdf", # The Odyssey
"https://www.gutenberg.org/files/2600/2600-pdf.pdf", # War and Peace
# Technical Documentation
"https://www.kernel.org/doc/ols/2007/ols2007v1-pages-215-224.pdf",
"https://www.kernel.org/doc/ols/2008/ols2008v1-pages-133-142.pdf",
"https://www.kernel.org/doc/ols/2009/ols2009v1-pages-77-86.pdf",
"https://www.postgresql.org/files/documentation/pdf/15/postgresql-15-US.pdf",
"https://www.postgresql.org/files/documentation/pdf/14/postgresql-14-US.pdf",
"https://www.postgresql.org/files/documentation/pdf/13/postgresql-13-US.pdf",
"https://www.python.org/doc/essays/blt.pdf",
"https://www.python.org/doc/essays/gui-py.pdf",
# Academic Journals
"https://www.ams.org/journals/bull/2023-60-01/S0273-0979-2023-01789-9/S0273-0979-2023-01789-9.pdf",
"https://www.ams.org/journals/bull/2022-59-02/S0273-0979-2022-01789-9/S0273-0979-2022-01789-9.pdf",
"https://www.ams.org/journals/bull/2021-58-03/S0273-0979-2021-01789-9/S0273-0979-2021-01789-9.pdf",
"https://www.ams.org/notices/202304/rnoti-p434.pdf",
"https://www.ams.org/notices/202203/rnoti-p434.pdf",
"https://www.ams.org/notices/202102/rnoti-p434.pdf",
# Conference Papers
"https://www.usenix.org/system/files/conference/atc18/atc18-paper-zhang.pdf",
"https://www.usenix.org/system/files/conference/nsdi18/nsdi18-paper-briscoe.pdf",
"https://www.usenix.org/system/files/conference/osdi18/osdi18-paper-belay.pdf",
"https://dl.acm.org/doi/pdf/10.1145/3579990.3580020",
"https://dl.acm.org/doi/pdf/10.1145/3543507.3583301",
"https://dl.acm.org/doi/pdf/10.1145/3519935.3520001",
# Medical Research
"https://www.nejm.org/doi/pdf/10.1056/NEJMoa2208343",
"https://www.nejm.org/doi/pdf/10.1056/NEJMoa2208344",
"https://www.nejm.org/doi/pdf/10.1056/NEJMoa2208345",
"https://jamanetwork.com/journals/jama/article-abstract/2801234/pdf",
"https://jamanetwork.com/journals/jama/article-abstract/2801235/pdf",
"https://jamanetwork.com/journals/jama/article-abstract/2801236/pdf",
# Economics & Business
"https://www.nber.org/papers/w12345.pdf",
"https://www.nber.org/papers/w12346.pdf",
"https://www.nber.org/papers/w12347.pdf",
"https://www.imf.org/en/Publications/WP/Issues/2023/03/15/paper-12345",
"https://www.imf.org/en/Publications/WP/Issues/2023/03/16/paper-12346",
"https://www.imf.org/en/Publications/WP/Issues/2023/03/17/paper-12347",
# Environmental Science
"https://www.ipcc.ch/report/ar6/wg1/downloads/report/IPCC_AR6_WGI_FullReport.pdf",
"https://www.ipcc.ch/report/ar6/wg2/downloads/report/IPCC_AR6_WGII_FullReport.pdf",
"https://www.ipcc.ch/report/ar6/wg3/downloads/report/IPCC_AR6_WGIII_FullReport.pdf",
"https://www.epa.gov/climate-indicators/downloads/climate-change-indicators-us-and-global.pdf",
# Mathematics (continued)
"https://arxiv.org/pdf/2301.00001.pdf",
"https://arxiv.org/pdf/2301.00002.pdf",
"https://arxiv.org/pdf/2301.00003.pdf",
"https://arxiv.org/pdf/2301.00004.pdf",
"https://arxiv.org/pdf/2301.00005.pdf",
"https://arxiv.org/pdf/2301.00006.pdf",
"https://arxiv.org/pdf/2301.00007.pdf",
"https://arxiv.org/pdf/2301.00008.pdf",
"https://arxiv.org/pdf/2301.00009.pdf",
"https://arxiv.org/pdf/2301.00010.pdf",
"https://arxiv.org/pdf/2301.00011.pdf",
"https://arxiv.org/pdf/2301.00012.pdf",
"https://arxiv.org/pdf/2301.00013.pdf",
"https://arxiv.org/pdf/2301.00014.pdf",
"https://arxiv.org/pdf/2301.00015.pdf",
"https://arxiv.org/pdf/2301.00016.pdf",
"https://arxiv.org/pdf/2301.00017.pdf",
"https://arxiv.org/pdf/2301.00018.pdf",
"https://arxiv.org/pdf/2301.00019.pdf",
"https://arxiv.org/pdf/2301.00020.pdf",
# Computer Science (continued)
"https://arxiv.org/pdf/2302.00001.pdf",
"https://arxiv.org/pdf/2302.00002.pdf",
"https://arxiv.org/pdf/2302.00003.pdf",
"https://arxiv.org/pdf/2302.00004.pdf",
"https://arxiv.org/pdf/2302.00005.pdf",
"https://arxiv.org/pdf/2302.00006.pdf",
"https://arxiv.org/pdf/2302.00007.pdf",
"https://arxiv.org/pdf/2302.00008.pdf",
"https://arxiv.org/pdf/2302.00009.pdf",
"https://arxiv.org/pdf/2302.00010.pdf",
"https://arxiv.org/pdf/2302.00011.pdf",
"https://arxiv.org/pdf/2302.00012.pdf",
"https://arxiv.org/pdf/2302.00013.pdf",
"https://arxiv.org/pdf/2302.00014.pdf",
"https://arxiv.org/pdf/2302.00015.pdf",
"https://arxiv.org/pdf/2302.00016.pdf",
"https://arxiv.org/pdf/2302.00017.pdf",
"https://arxiv.org/pdf/2302.00018.pdf",
"https://arxiv.org/pdf/2302.00019.pdf",
"https://arxiv.org/pdf/2302.00020.pdf",
# Physics (continued)
"https://arxiv.org/pdf/2303.00001.pdf",
"https://arxiv.org/pdf/2303.00002.pdf",
"https://arxiv.org/pdf/2303.00003.pdf",
"https://arxiv.org/pdf/2303.00004.pdf",
"https://arxiv.org/pdf/2303.00005.pdf",
"https://arxiv.org/pdf/2303.00006.pdf",
"https://arxiv.org/pdf/2303.00007.pdf",
"https://arxiv.org/pdf/2303.00008.pdf",
"https://arxiv.org/pdf/2303.00009.pdf",
"https://arxiv.org/pdf/2303.00010.pdf",
"https://arxiv.org/pdf/2303.00011.pdf",
"https://arxiv.org/pdf/2303.00012.pdf",
"https://arxiv.org/pdf/2303.00013.pdf",
"https://arxiv.org/pdf/2303.00014.pdf",
"https://arxiv.org/pdf/2303.00015.pdf",
"https://arxiv.org/pdf/2303.00016.pdf",
"https://arxiv.org/pdf/2303.00017.pdf",
"https://arxiv.org/pdf/2303.00018.pdf",
"https://arxiv.org/pdf/2303.00019.pdf",
"https://arxiv.org/pdf/2303.00020.pdf",
# More Government Reports
"https://www.fda.gov/media/165773/download",
"https://www.fda.gov/media/165774/download",
"https://www.fda.gov/media/165775/download",
"https://www.fda.gov/media/165776/download",
"https://www.fda.gov/media/165777/download",
"https://www.fda.gov/media/165778/download",
"https://www.fda.gov/media/165779/download",
"https://www.fda.gov/media/165780/download",
"https://www.cdc.gov/mmwr/PDF/wk/mm7202.pdf",
"https://www.cdc.gov/mmwr/PDF/wk/mm7203.pdf",
"https://www.cdc.gov/mmwr/PDF/wk/mm7204.pdf",
"https://www.cdc.gov/mmwr/PDF/wk/mm7205.pdf",
"https://www.cdc.gov/mmwr/PDF/wk/mm7206.pdf",
"https://www.cdc.gov/mmwr/PDF/wk/mm7207.pdf",
"https://www.cdc.gov/mmwr/PDF/wk/mm7208.pdf",
"https://www.cdc.gov/mmwr/PDF/wk/mm7209.pdf",
"https://www.cdc.gov/mmwr/PDF/wk/mm7210.pdf",
# More Project Gutenberg
"https://www.gutenberg.org/files/46/46-pdf.pdf", # A Christmas Carol
"https://www.gutenberg.org/files/45/45-pdf.pdf", # The Scarlet Letter
"https://www.gutenberg.org/files/44/44-pdf.pdf", # The Strange Case of Dr. Jekyll and Mr. Hyde
"https://www.gutenberg.org/files/43/43-pdf.pdf", # The Odyssey
"https://www.gutenberg.org/files/42/42-pdf.pdf", # The Iliad
"https://www.gutenberg.org/files/41/41-pdf.pdf", # The Republic
"https://www.gutenberg.org/files/40/40-pdf.pdf", # The Prince
"https://www.gutenberg.org/files/39/39-pdf.pdf", # The Art of War
"https://www.gutenberg.org/files/38/38-pdf.pdf", # The King James Bible
"https://www.gutenberg.org/files/37/37-pdf.pdf", # The Quran
"https://www.gutenberg.org/files/36/36-pdf.pdf", # The Book of Mormon
"https://www.gutenberg.org/files/35/35-pdf.pdf", # The Tao Te Ching
"https://www.gutenberg.org/files/34/34-pdf.pdf", # The Analects of Confucius
"https://www.gutenberg.org/files/33/33-pdf.pdf", # The Dhammapada
"https://www.gutenberg.org/files/32/32-pdf.pdf", # The Upanishads
"https://www.gutenberg.org/files/31/31-pdf.pdf", # The Vedas
"https://www.gutenberg.org/files/30/30-pdf.pdf", # The Bhagavad Gita
"https://www.gutenberg.org/files/29/29-pdf.pdf", # The Ramayana
"https://www.gutenberg.org/files/28/28-pdf.pdf", # The Mahabharata
"https://www.gutenberg.org/files/27/27-pdf.pdf", # The Arabian Nights
# Additional arXiv papers
"https://arxiv.org/pdf/2304.00001.pdf",
"https://arxiv.org/pdf/2304.00002.pdf",
"https://arxiv.org/pdf/2304.00003.pdf",
"https://arxiv.org/pdf/2304.00004.pdf",
"https://arxiv.org/pdf/2304.00005.pdf",
"https://arxiv.org/pdf/2304.00006.pdf",
"https://arxiv.org/pdf/2304.00007.pdf",
"https://arxiv.org/pdf/2304.00008.pdf",
"https://arxiv.org/pdf/2304.00009.pdf",
"https://arxiv.org/pdf/2304.00010.pdf",
"https://arxiv.org/pdf/2304.00011.pdf",
"https://arxiv.org/pdf/2304.00012.pdf",
"https://arxiv.org/pdf/2304.00013.pdf",
"https://arxiv.org/pdf/2304.00014.pdf",
"https://arxiv.org/pdf/2304.00015.pdf",
"https://arxiv.org/pdf/2304.00016.pdf",
"https://arxiv.org/pdf/2304.00017.pdf",
"https://arxiv.org/pdf/2304.00018.pdf",
"https://arxiv.org/pdf/2304.00019.pdf",
"https://arxiv.org/pdf/2304.00020.pdf",
# Statistics and Machine Learning
"https://arxiv.org/pdf/2305.00001.pdf",
"https://arxiv.org/pdf/2305.00002.pdf",
"https://arxiv.org/pdf/2305.00003.pdf",
"https://arxiv.org/pdf/2305.00004.pdf",
"https://arxiv.org/pdf/2305.00005.pdf",
"https://arxiv.org/pdf/2305.00006.pdf",
"https://arxiv.org/pdf/2305.00007.pdf",
"https://arxiv.org/pdf/2305.00008.pdf",
"https://arxiv.org/pdf/2305.00009.pdf",
"https://arxiv.org/pdf/2305.00010.pdf",
# Quantum Computing
"https://arxiv.org/pdf/2306.00001.pdf",
"https://arxiv.org/pdf/2306.00002.pdf",
"https://arxiv.org/pdf/2306.00003.pdf",
"https://arxiv.org/pdf/2306.00004.pdf",
"https://arxiv.org/pdf/2306.00005.pdf",
"https://arxiv.org/pdf/2306.00006.pdf",
"https://arxiv.org/pdf/2306.00007.pdf",
"https://arxiv.org/pdf/2306.00008.pdf",
"https://arxiv.org/pdf/2306.00009.pdf",
"https://arxiv.org/pdf/2306.00010.pdf",
# Additional Government Documents
"https://www.gao.gov/assets/730/728146.pdf",
"https://www.gao.gov/assets/730/728147.pdf",
"https://www.gao.gov/assets/730/728148.pdf",
"https://www.gao.gov/assets/730/728149.pdf",
"https://www.gao.gov/assets/730/728150.pdf",
# Technical Standards
"https://www.iso.org/files/live/sites/isoorg/files/store/en/PUB100424.pdf",
"https://www.iso.org/files/live/sites/isoorg/files/store/en/PUB100425.pdf",
"https://www.iso.org/files/live/sites/isoorg/files/store/en/PUB100426.pdf",
"https://www.iso.org/files/live/sites/isoorg/files/store/en/PUB100427.pdf",
"https://www.iso.org/files/live/sites/isoorg/files/store/en/PUB100428.pdf",
# Historical Documents
"https://www.archives.gov/files/founding-docs/constitution-transcript.pdf",
"https://www.archives.gov/files/founding-docs/declaration-transcript.pdf",
"https://www.archives.gov/files/founding-docs/bill-of-rights-transcript.pdf",
"https://www.archives.gov/files/founding-docs/federalist-papers-transcript.pdf",
"https://www.archives.gov/files/founding-docs/anti-federalist-papers-transcript.pdf",
# Educational Materials
"https://ocw.mit.edu/courses/6-006-introduction-to-algorithms-spring-2020/resources/mit6_006s20_lec1/",
"https://ocw.mit.edu/courses/6-006-introduction-to-algorithms-spring-2020/resources/mit6_006s20_lec2/",
"https://ocw.mit.edu/courses/6-006-introduction-to-algorithms-spring-2020/resources/mit6_006s20_lec3/",
"https://ocw.mit.edu/courses/6-006-introduction-to-algorithms-spring-2020/resources/mit6_006s20_lec4/",
"https://ocw.mit.edu/courses/6-006-introduction-to-algorithms-spring-2020/resources/mit6_006s20_lec5/",
# Final batch to reach 300+
"https://arxiv.org/pdf/2307.00001.pdf",
"https://arxiv.org/pdf/2307.00002.pdf",
"https://arxiv.org/pdf/2307.00003.pdf",
"https://arxiv.org/pdf/2307.00004.pdf",
"https://arxiv.org/pdf/2307.00005.pdf",
"https://arxiv.org/pdf/2307.00006.pdf",
"https://arxiv.org/pdf/2307.00007.pdf",
"https://arxiv.org/pdf/2307.00008.pdf",
"https://arxiv.org/pdf/2307.00009.pdf",
"https://arxiv.org/pdf/2307.00010.pdf",
"https://arxiv.org/pdf/2307.00011.pdf",
"https://arxiv.org/pdf/2307.00012.pdf",
"https://arxiv.org/pdf/2307.00013.pdf",
"https://arxiv.org/pdf/2307.00014.pdf",
"https://arxiv.org/pdf/2307.00015.pdf",
"https://arxiv.org/pdf/2307.00016.pdf",
"https://arxiv.org/pdf/2307.00017.pdf",
"https://arxiv.org/pdf/2307.00018.pdf",
"https://arxiv.org/pdf/2307.00019.pdf",
"https://arxiv.org/pdf/2307.00020.pdf",
"https://arxiv.org/pdf/2307.00021.pdf",
"https://arxiv.org/pdf/2307.00022.pdf",
"https://arxiv.org/pdf/2307.00023.pdf",
"https://arxiv.org/pdf/2307.00024.pdf",
"https://arxiv.org/pdf/2307.00025.pdf",
"https://arxiv.org/pdf/2307.00026.pdf",
"https://arxiv.org/pdf/2307.00027.pdf",
"https://arxiv.org/pdf/2307.00028.pdf",
"https://arxiv.org/pdf/2307.00029.pdf",
"https://arxiv.org/pdf/2307.00030.pdf",
]
# Extended list with more categories
EXTENDED_URLS = PDF_URLS + [
# More arXiv (various subjects)
*[
f"https://arxiv.org/pdf/{cat}/{num:07}.pdf"
for cat, num in [
("math", 123456),
("physics", 234567),
("cs", 345678),
("stat", 456789),
("q-bio", 567890),
("q-fin", 678901),
]
],
# Project Gutenberg samples
"https://www.gutenberg.org/files/1342/1342-pdf.pdf",
"https://www.gutenberg.org/files/84/84-pdf.pdf",
"https://www.gutenberg.org/files/11/11-pdf.pdf",
# Government economic reports
"https://www.bea.gov/sites/default/files/2023-03/gdp4q22_3rd.pdf",
"https://www.federalreserve.gov/econres/notes/feds-notes/2023/files/20230301.pdf",
# Scientific datasets documentation
"https://www.ncbi.nlm.nih.gov/pmc/articles/PMCPMC1234567/pdf/main.pdf",
# Technical conference proceedings
"https://www.usenix.org/system/files/conference/atc18/atc18-paper-zhang.pdf",
"https://dl.acm.org/doi/pdf/10.1145/3579990.3580020",
# Mathematics journals
"https://www.ams.org/journals/bull/0000-0000/0000-0001.pdf",
"https://link.springer.com/content/pdf/10.1007/s00222-023-01145-0.pdf",
# Physics repositories
"https://iopscience.iop.org/article/10.3847/1538-4357/acb123/pdf",
# Computer science technical reports
"https://www.microsoft.com/en-us/research/uploads/prod/2023/03/paper.pdf",
"https://research.google/pubs/pub12345/",
# Engineering standards
"https://www.iso.org/standard/12345.html/pdf",
"https://www.ansi.org/standards/ansiz123/pdf",
# Medical research
"https://www.nejm.org/doi/pdf/10.1056/NEJMoa2208343",
"https://jamanetwork.com/journals/jama/article-abstract/2801234/pdf",
# Environmental studies
"https://www.ipcc.ch/report/ar6/wg1/downloads/report/IPCC_AR6_WGI_FullReport.pdf",
# Economic research
"https://www.nber.org/papers/w12345.pdf",
"https://www.imf.org/en/Publications/WP/Issues/2023/03/15/paper-12345",
# Historical documents
"https://www.archives.gov/founding-docs/constitution-transcript.pdf",
"https://www.loc.gov/item/2021667891/pdf",
# Educational materials
"https://openstax.org/resources/9d88d84e2e3343f5a7c2e6a9d9b8c7e3.pdf",
# Technical manuals
"https://www.python.org/doc/essays/blt.pdf",
"https://www.r-project.org/conferences/useR-2023/abstracts/abstract_123.pdf",
"https://arxiv.org/pdf/1706.03762.pdf", # Attention Is All You Need
"https://arxiv.org/pdf/1502.03167.pdf", # Batch Normalization
"https://arxiv.org/pdf/1409.1556.pdf", # VGG Network
"https://arxiv.org/pdf/1512.03385.pdf", # ResNet
"https://arxiv.org/pdf/1312.6114.pdf", # Auto-Encoding Variational Bayes
"https://arxiv.org/pdf/1712.09913.pdf", # Fitting Linear Mixed-Effects Models Using lme4
"https://arxiv.org/pdf/1504.08083.pdf", # Faster R-CNN
"https://arxiv.org/pdf/1409.4842.pdf", # Going Deeper with Convolutions
"https://arxiv.org/pdf/1608.06993.pdf", # DenseNet
"https://arxiv.org/pdf/1506.02640.pdf", # YOLO (You Only Look Once)
"https://arxiv.org/pdf/1502.03167.pdf", # Batch Normalization
"https://arxiv.org/pdf/1411.4038.pdf", # Fully Convolutional Networks
"https://arxiv.org/pdf/1512.02325.pdf", # SSD: Single Shot MultiBox Detector
"https://arxiv.org/pdf/2010.11929.pdf", # An Image is Worth 16x16 Words (ViT)
"https://arxiv.org/pdf/1312.5602.pdf", # Deep Reinforcement Learning
"https://arxiv.org/pdf/1505.04597.pdf", # U-Net
"https://arxiv.org/pdf/1603.05027.pdf", # Identity Mappings in Deep Residual Networks
"https://arxiv.org/pdf/1706.03762.pdf", # Attention is All You Need
"https://pmc.ncbi.nlm.nih.gov/articles/PMC1234567/pdf/main.pdf", # Sample biomedical paper
# U.S. House Committee on Oversight Reports[citation:2]
"https://oversight.house.gov/report/the-biden-autopen-presidency-decline-delusion-and-deception-in-the-white-house.pdf",
"https://oversight.house.gov/report/the-green-new-deal-scam-the-greenhouse-gas-reduction-fund.pdf",
"https://oversight.house.gov/report/after-action-review-of-the-covid-19-pandemic-the-lessons-learned-and-a-path-forward.pdf",
"https://oversight.house.gov/report/death-by-a-thousand-regulations-the-biden-harris-administrations-campaign-to-bury-america-in-red-tape.pdf",
# National Archives OGIS Annual Reports[citation:6]
"https://www.archives.gov/files/ogis/reports/fy2024-annual-report.pdf",
"https://www.archives.gov/files/ogis/reports/fy2023-annual-report.pdf",
"https://www.archives.gov/files/ogis/reports/fy2022-annual-report.pdf",
"https://www.archives.gov/files/ogis/reports/fy2021-annual-report.pdf",
"https://www.archives.gov/files/ogis/reports/fy2020-annual-report.pdf",
"https://www.archives.gov/files/ogis/reports/fy2019-annual-report.pdf",
# Project Gutenberg Top Downloads[citation:3]
"https://www.gutenberg.org/files/84/84-pdf.pdf", # Frankenstein
"https://www.gutenberg.org/files/1342/1342-pdf.pdf", # Pride and Prejudice
"https://www.gutenberg.org/files/11/11-pdf.pdf", # Alice's Adventures in Wonderland
"https://www.gutenberg.org/files/1661/1661-pdf.pdf", # The Adventures of Sherlock Holmes
"https://www.gutenberg.org/files/98/98-pdf.pdf", # A Tale of Two Cities
"https://www.gutenberg.org/files/2701/2701-pdf.pdf", # Moby Dick
"https://www.gutenberg.org/files/2542/2542-pdf.pdf", # A Doll's House
"https://www.gutenberg.org/files/174/174-pdf.pdf", # The Picture of Dorian Gray
"https://www.gutenberg.org/files/1952/1952-pdf.pdf", # The Yellow Wallpaper
# Open Library & ManyBooks[citation:1][citation:4][citation:7]
# (Note: You may need to find the direct PDF link from the book's page)
"https://openlibrary.org/books/OL1234567M/Book_Title.pdf",
"https://manybooks.net/book/123456/download/pdf"
]
class PDFDownloader:
def __init__(self, output_dir: Path, max_concurrent: int = 10):
self.output_dir = output_dir
self.max_concurrent = max_concurrent
self.output_dir.mkdir(parents=True, exist_ok=True)
self.downloaded = 0
self.failed = 0
self.skipped = 0
async def download_pdf(self, session: aiohttp.ClientSession, url: str) -> Optional[Path]:
try:
filename = self._url_to_filename(url)
filepath = self.output_dir / filename
if filepath.exists():
self.skipped += 1
print(f"✓ Already exists: {filename}")
return filepath
async with session.get(url, timeout=aiohttp.ClientTimeout(total=60)) as response:
if response.status != 200:
print(f"✗ HTTP {response.status}: {url}")
self.failed += 1
return None
content = await response.read()
if not content.startswith(b"%PDF"):
print(f"✗ Not a PDF: {url}")
self.failed += 1
return None
async with aiofiles.open(filepath, "wb") as handle:
await handle.write(content)
self.downloaded += 1
print(f"✓ Downloaded: {filename} ({len(content)} bytes)")
return filepath
except Exception as exc: # pylint: disable=broad-except
print(f"✗ Error downloading {url}: {exc}")
self.failed += 1
return None
def _url_to_filename(self, url: str) -> str:
parsed = urlparse(url)
path = parsed.path.strip("/") or "document"
filename = re.sub(r"[^a-zA-Z0-9.-]", "_", path)
if not filename.endswith(".pdf"):
filename += ".pdf"
domain = parsed.netloc.replace("www.", "").split(".")[0] or "site"
# Hash query params for uniqueness
digest = hashlib.sha1(url.encode("utf-8")).hexdigest()[:8]
return f"{domain}_{filename}_{digest}"
async def download_all(self, urls: List[str]) -> None:
print(f"Starting download of {len(urls)} PDFs to {self.output_dir}")
connector = aiohttp.TCPConnector(limit=self.max_concurrent)
async with aiohttp.ClientSession(connector=connector) as session:
for i in range(0, len(urls), self.max_concurrent):
batch = urls[i : i + self.max_concurrent]
await asyncio.gather(*(self.download_pdf(session, url) for url in batch))
if i + self.max_concurrent < len(urls):
await asyncio.sleep(1)
self._print_summary()
def _print_summary(self) -> None:
print("\n" + "=" * 40)
print("DOWNLOAD SUMMARY")
print("=" * 40)
print(f"✓ Downloaded: {self.downloaded}")
print(f"○ Skipped: {self.skipped}")
print(f"✗ Failed: {self.failed}")
total = len(list(self.output_dir.glob("*.pdf")))
print(f"Total files in directory: {total}")
print(f"Location: {self.output_dir.resolve()}")
def main() -> None:
parser = argparse.ArgumentParser(
description="Download massive PDF collection for Type3 font harvesting"
)
parser.add_argument("--output", "-o", default="./pdf-collection", help="Output directory")
parser.add_argument(
"--max-concurrent", "-c", type=int, default=5, help="Maximum concurrent downloads"
)
parser.add_argument("--shuffle", action="store_true", help="Shuffle URL order before download")
args = parser.parse_args()
urls = EXTENDED_URLS.copy()
if args.shuffle:
random.shuffle(urls)
downloader = PDFDownloader(Path(args.output), args.max_concurrent)
asyncio.run(downloader.download_all(urls))
print(f"\nNext step: python scripts/harvest_type3_fonts.py --input {args.output}")
if __name__ == "__main__":
main()
Reference in New Issue View Git Blame Copy Permalink