diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 0000000..4d9994b --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,21 @@ +### Describe the issue you are having + + +### Describe your setup? + + +### Are you running sniproxy + + +### DNS Configuration +``` + +``` + +### Sniproxy output +Please paste the output from `docker logs | sed 's/.*\:443 \[//;s/\].*//' | sort | uniq -c` below +``` + +``` diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..cbabb85 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,25 @@ +### What CDN does this PR relate to + + +### Does this require running via sniproxy + + +### Capture method + + +### Testing Scenario + + +### Testing Configuration +``` + +``` + +### Sniproxy output +Please paste the output from `docker logs | sed 's/.*\:443 \[//;s/\].*//' | sort | uniq -c` below +``` + +``` + diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..86af95d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +scripts/output diff --git a/README.md b/README.md index c20e846..d9d606b 100644 --- a/README.md +++ b/README.md @@ -11,16 +11,60 @@ You can use this list one of two ways: - Overriding DNS for these hostnames to point to the IP of your cache server. - Use them in Squid with WCCP to redirect content to the right cache server. -There is a separate file for each caching service. Some notes on formatting: +There is a cache_domains.json file to define CDNs and additional meta deta with the following structure + +- cache_domains: Array of cache_domain object + - name: shortname for the cache domain + - description: a longer description to aid others in identifying what this domain does (not all users of this repo will want to enable all caches) + - notes: implementation specific notes which may be useful for other users + - mixed_content: true if this domain hosts mixed https and http content (a straight dns injection is unlikely to work in this case). Assumed to be false if undefined + - domain_files: array of files within the repo assosciated to the cdn. Most cdn's only need one file + - Example domain entry for origin +```json +{ + "cache_domains": [ + { + "name": "origin", + "description": "CDN for origin", + "notes": "Should be enabled for HTTP traffic only or with a HTTPS proxy else origin client download fails", + "mixed_content": true, + "domain_files": ["origin.txt"] + } + ] +} +``` + +### Host file format + +There is a separate file for each cacheable service. Some notes on formatting: - Every line should be a seperate hostname for that service. - - Wildcards can be represented with a space. + - Only one entry is permitted per line. + - Wildcards are permitted as per below - Lines starting with a # will be treated as a comment. + - Files must end with an empty newline. + +#### Wildcards + +The wildcard format shall be defined as per the below + + - Wildcards should be represented with an asterix. + - If a wildcard is used, it should be the first character on the line. + - Wildcards are not treated as matching null, e.g. `*.example.com` will match `a.example.com` but will not match `example.com` + - Only simple domain wildcards will be accepted eg `*.example.com` not `*ww.example.com` + +##### Notes for Squid users + +If you are using these files within a squid dst_domain acl you will need to reformat the wildcard entries to be compliant with the squid acl format. The following regex should suffice `s/*\./\./` ## Updates Please fork this repository and submit pull requests if you have any extra hostnames or services to add. We want this list to be definitive and collaborative! +### New CDNs + +If the CDN you are adding does not already exist please remember to add an entry within cache_domains.json as well as a new .txt file + ## Issues and Feedback Please raise all issues and feedback on GitHub at [uklans/cache-domains](https://github.com/uklans/cache-domains/issues). @@ -47,4 +91,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file +SOFTWARE. diff --git a/blizzard.txt b/blizzard.txt index d52f4d7..7d85419 100644 --- a/blizzard.txt +++ b/blizzard.txt @@ -9,4 +9,6 @@ blzddist3-a.akamaihd.net blzddist4-a.akamaihd.net level3.blizzard.com nydus.battle.net -edge.blizzard.top.comcast.net \ No newline at end of file +edge.blizzard.top.comcast.net +cdn.blizzard.com +*.cdn.blizzard.com diff --git a/cache_domains.json b/cache_domains.json new file mode 100644 index 0000000..699145c --- /dev/null +++ b/cache_domains.json @@ -0,0 +1,107 @@ +{ + "cache_domains": [ + { + "name": "apple", + "description": "CDN for apple", + "domain_files": ["apple.txt"] + }, + { + "name": "arenanet", + "description": "CDN for guild wars, HoT", + "domain_files": ["arenanet.txt"] + }, + { + "name": "blizzard", + "description": "CDN for blizzard/battle.net", + "domain_files": ["blizzard.txt"] + }, + { + "name": "daybreak", + "description": "Daybreak games CDN", + "domain_files": ["daybreak.txt"] + }, + { + "name": "frontier", + "description": "CDN for frontier games", + "domain_files": ["frontier.txt"] + }, + { + "name": "hirez", + "description": "CDN for hirez", + "domain_files": ["hirez.txt"] + }, + { + "name": "minecraft", + "description": "CDN for minecraft client and updater", + "domain_files": ["minecraft.txt"] + }, + { + "name": "nexusmods", + "description": "Nexus mods / skyrim content", + "domain_files": ["nexusmods.txt"] + }, + { + "name": "nintendo", + "description": "CDN for nintendo (swtich)", + "domain_files": ["nintendo.txt"] + }, + { + "name": "origin", + "description": "CDN for origin", + "notes": "Should be enabled for HTTP traffic only or with a HTTPS proxy else origin client download fails", + "mixed_content": true, + "domain_files": ["origin.txt"] + }, + { + "name": "renegadex", + "description": "CDN for Renegade X", + "domain_files": ["renegadex.txt"] + }, + { + "name": "riot", + "description": "CDN for riot games", + "domain_files": ["riot.txt"] + }, + { + "name": "rockstar", + "description": "CDN for rockstar games", + "domain_files": ["rockstar.txt"] + }, + { + "name": "sony", + "description": "CDN for sony / playstation", + "domain_files": ["sony.txt"] + }, + { + "name": "steam", + "description": "CDN for steam platform", + "domain_files": ["steam.txt"] + }, + { + "name": "uplay", + "description": "CDN for uplay downloader", + "domain_files": ["uplay.txt"] + }, + { + "name": "twitch", + "description": "CDN for twitch games / mods and addons", + "domain_files": ["twitchapp.txt"] + }, + { + "name": "wargaming", + "description": "CDN for wargaming.net", + "domain_files": ["wargaming.net.txt"] + }, + { + "name": "wsus", + "description": "CDN for windows updates", + "domain_files": ["windowsupdates.txt"] + }, + { + "name": "xboxlive", + "description": "CDN for xboxlive", + "domain_files": ["xboxlive.txt"] + } + ] +} + diff --git a/daybreak.txt b/daybreak.txt new file mode 100644 index 0000000..56c62d0 --- /dev/null +++ b/daybreak.txt @@ -0,0 +1,6 @@ +# Daybreak games +# +# Do NOT cache manifest.patch.daybreakgames.com +# +#PS2 +pls.patch.daybreakgames.com diff --git a/gog.txt b/gog.txt deleted file mode 100644 index e67f9aa..0000000 --- a/gog.txt +++ /dev/null @@ -1,4 +0,0 @@ -cdn.gog.com -wpc.11df.deltacdn.net -11df-eu-lb.wpc.edgecastcdn.net -11df-eu-lb.apr-11df.edgecastdns.net \ No newline at end of file diff --git a/nintendo.txt b/nintendo.txt index 35d5e06..0f9de4b 100644 --- a/nintendo.txt +++ b/nintendo.txt @@ -1 +1,8 @@ -ccs.cdn.wup.shop.nintendo.com \ No newline at end of file +ccs.cdn.wup.shop.nintendo.com +pushmo.hac.lp1.eshop.nintendo.net +ecs-lp1.hac.shop.nintendo.net +receive-lp1.dg.srv.nintendo.net +aqua.hac.lp1.d4c.nintendo.net +atum.hac.lp1.d4c.nintendo.net +bugyo.hac.lp1.eshop.nintendo.net +tagaya.hac.lp1.eshop.nintendo.net diff --git a/origin.txt b/origin.txt index 25696bd..2d07963 100644 --- a/origin.txt +++ b/origin.txt @@ -2,4 +2,5 @@ origin-a.akamaihd.net akamai.cdn.ea.com lvlt.cdn.ea.com -river.data.ea.com \ No newline at end of file +river.data.ea.com +origin-a.akamaihd.net.edgesuite.net diff --git a/renegadex.txt b/renegadex.txt new file mode 100644 index 0000000..fa42197 --- /dev/null +++ b/renegadex.txt @@ -0,0 +1,12 @@ +rxp-fl.cncirc.net +rxp-chi.cncirc.net +rxp-nz.cncirc.net +rxp-bgr.cncirc.net +rxp-fr.cncirc.net +rxp-nyc.cncirc.net +rxp-uk.cncirc.net +rxp-sg.cncirc.net +rxp-la.cncirc.net +rxp-fin.cncirc.net +denver1.renegade-x.com +seattle1.renegade-x.com \ No newline at end of file diff --git a/riot.txt b/riot.txt index c3451b2..c3e26a1 100644 --- a/riot.txt +++ b/riot.txt @@ -1,2 +1,4 @@ l3cdn.riotgames.com -worldwide.l3cdn.riotgames.com \ No newline at end of file +worldwide.l3cdn.riotgames.com +riotgamespatcher-a.akamaihd.net +riotgamespatcher-a.akamaihd.net.edgesuite.net diff --git a/scripts/config.example.json b/scripts/config.example.json new file mode 100644 index 0000000..6d58ee4 --- /dev/null +++ b/scripts/config.example.json @@ -0,0 +1,19 @@ +{ + "ips": { + "steam": "10.10.3.11", + "origin": "10.10.3.12", + "blizzard": "10.10.3.13", + "windows": "10.10.3.14", + "riot": "10.10.3.15", + "generic": "10.10.3.16" + }, + "cache_domains": { + "default": "generic", + "blizzard": "blizzard", + "origin": "origin", + "riot": "riot", + "steam": "steam", + "wsus": "windows", + "xboxlive": "windows" + } +} diff --git a/scripts/create-unbound.sh b/scripts/create-unbound.sh new file mode 100755 index 0000000..c0d4c0e --- /dev/null +++ b/scripts/create-unbound.sh @@ -0,0 +1,63 @@ +#!/bin/bash +basedir=".." +outputdir="output/unbound" +path="${basedir}/cache_domains.json" + +export IFS=' ' + +test=$(which jq); +out=$? +if [ $out -gt 0 ] ; then + echo "This script requires jq to be installed." + echo "Your package manager should be able to find it" + exit 1 +fi + +cachenamedefault="disabled" + +while read line; do + ip=$(jq -r ".ips[\"${line}\"]" config.json) + declare "cacheip$line"="$ip" +done <<< $(jq -r '.ips | to_entries[] | .key' config.json) + +while read line; do + name=$(jq -r ".cache_domains[\"${line}\"]" config.json) + declare "cachename$line"="$name" +done <<< $(jq -r '.cache_domains | to_entries[] | .key' config.json) + +rm -rf ${outputdir} +mkdir -p ${outputdir} +while read entry; do + unset cacheip + unset cachename + key=$(jq -r ".cache_domains[$entry].name" $path) + cachename="cachename${key}" + if [ -z "${!cachename}" ]; then + cachename="cachenamedefault" + fi + if [[ ${!cachename} == "disabled" ]]; then + continue; + fi + cacheipname="cacheip${!cachename}" + cacheip=${!cacheipname} + while read fileid; do + while read filename; do + destfilename=$(echo $filename | sed -e 's/txt/conf/') + outputfile=${outputdir}/${destfilename} + touch $outputfile + echo "server:" >> $outputfile + while read fileentry; do + # Ignore comments + if [[ $fileentry == \#* ]]; then + continue + fi + parsed=$(echo $fileentry | sed -e "s/^\*\.//") + if grep -q "$parsed" $outputfile; then + continue + fi + echo " local-zone: \"${parsed}\" redirect" >> $outputfile + echo " local-data: \"${parsed} 30 IN A ${cacheip}\"" >> $outputfile + done <<< $(cat ${basedir}/$filename); + done <<< $(jq -r ".cache_domains[$entry].domain_files[$fileid]" $path) + done <<< $(jq -r ".cache_domains[$entry].domain_files | to_entries[] | .key" $path) +done <<< $(jq -r '.cache_domains | to_entries[] | .key' $path) diff --git a/steam.txt b/steam.txt index ceb13f1..3d2756d 100644 --- a/steam.txt +++ b/steam.txt @@ -16,4 +16,18 @@ content-origin.steampowered.com clientconfig.akamai.steamtransparent.com steampipe.akamaized.net edgecast.steamstatic.com -steam.apac.qtlglb.com.mwcloudcdn.com \ No newline at end of file +steam.apac.qtlglb.com.mwcloudcdn.com +*.cs.steampowered.com +*.edgecast.steamstatic.com +*.steamcontent.com +cdn1-sea1.valve.net +cdn2-sea1.valve.net +*.steam-content-dnld-1.apac-1-cdn.cqloud.com +steam.apac.qtlglb.com +edge.steam-dns.top.comcast.net +edge.steam-dns-2.top.comcast.net +steamcdn-a.akamaihd.net +steam.naeu.qtlglb.com +steampipe-kr.akamaized.net +steam.ix.asn.au +steam.eca.qtlglb.com diff --git a/twitchapp.txt b/twitchapp.txt new file mode 100644 index 0000000..8f18078 --- /dev/null +++ b/twitchapp.txt @@ -0,0 +1,4 @@ +d3rmjivj4k4f0t.cloudfront.net +addons.forgesvc.net +media.forgecdn.net +files.forgecdn.net diff --git a/uplay.txt b/uplay.txt index 610aa07..4042a60 100644 --- a/uplay.txt +++ b/uplay.txt @@ -1 +1 @@ -.cdn.ubi.com +*.cdn.ubi.com diff --git a/wargaming.net.txt b/wargaming.net.txt index 6c3160e..4f98e1d 100644 --- a/wargaming.net.txt +++ b/wargaming.net.txt @@ -5,4 +5,5 @@ wargaming.net.edgesuite.ne wgusst-na.wargaming.net wgusst-eu.wargaming.net update-v4r4h10x.worldofwarships.com -dl-wows-ak.wargaming.net \ No newline at end of file +dl-wows-ak.wargaming.net +wgus-wotasia.wargaming.net diff --git a/windowsupdates.txt b/windowsupdates.txt index 2738150..a6327ce 100644 --- a/windowsupdates.txt +++ b/windowsupdates.txt @@ -1,2 +1,8 @@ -download.windowsupdate.com -officecdn.microsoft.com \ No newline at end of file +officecdn.microsoft.com +*.windowsupdate.com +windowsupdate.com +*.dl.delivery.mp.microsoft.com +dl.delivery.mp.microsoft.com +*.update.microsoft.com +*.do.dsp.mp.microsoft.com +*.microsoft.com.edgesuite.net