-- (c) 2007 Nymbia.  see LGPLv2.1.txt for full details.
--this tool is run in the lua command line.  http://lua.org
--socket is required for internet data.
--get socket here: http://luaforge.net/projects/luasocket/
local starttime = os.time()
debug = 1
local function dprint(dlevel, ...)
	if dlevel and debug >= dlevel then
		print(...)
	end
end

http = require("socket.http")
url = require("socket.url")
local httptime = 0
local function getpage(...)
	local stime = os.time()
	local r = http.request(...)
	httptime = httptime + (os.time() - stime)
	return r
end
local skip = ""
local sets = {}
local results = {}
local f = assert(io.open("data.lua", "r"))
local file = f:read("*all")
local notmined = {}
f:close()
local setcount = 0
for set, data in file:gmatch('\t%[%"([^"]+)%"%][^=]-= "([^"]-)"') do
	sets[set] = data
	setcount = setcount + 1
end
local setid = 0
for set, data in pairs(sets) do
	setid = setid + 1
	dprint(1, ("current set: %4d/%4d"):format(setid, setcount), set)
	local newset
	if data:sub(1,2) ~= "m," then
		if set:match("^InstanceLoot%.") then
			local boss = set:match("%.([^%.]+)$")
			if boss == " Smite" then
				boss = "Mr. Smite"
			end
			local page = getpage("http://www.wowhead.com/?npcs&filter=na="..url.escape(boss))
			local matchy = (page:match("%'%&filter%=na%=([^%)]+)'%)")):gsub("([%.%(%)%%%+%-%*%?%[%]%^%$])", "%%%1")
			local id = page:match("%['"..matchy.."'[^\n^,]-,[^\n^,]-,'[^\n^,]-',(%d+),")
			if id then
				page = getpage("http://www.wowhead.com/?npc="..id)
				local count = 0
				local normaldropsstring,heroicdropsstring = page:match("'Normal drops',(%b[]),[^\n]+,'Heroic drops',(%b[])")
				if not heroicdropsstring then
					heroicdropsstring = string.match(page, "'Heroic drops',%b[]")
				end
				if heroicdropsstring then
					if normaldropssstring then
						for itemid, quality, droprate in string.gmatch(normaldropsstring, "%[%'%d[^\n^,]+',[^\n^,]-,[^\n^,]-,(%d+),[^\n^,]-,%[[^\n^%]]-b class%=%A%a(%d)[^%]^\n]-%],[^\n^,]-,[^\n^,]-,([^\n^,]-),[^\n^,]-]") do
							droprate = math.floor(tonumber(droprate)*10)
							if tonumber(quality) > 0 and (droprate > 50 or tonumber(quality) > 2) then
								if newset then
									newset = newset..","..itemid..":"..droprate
								else
									newset = itemid..":"..droprate
								end
								count = count + 1
							end
						end
					end
					local normalsub = set:match("^InstanceLoot%.(.+)$")
					local heroicname = "InstanceLootHeroic."..normalsub
					local heroicstr
					for itemid, quality, droprate in string.gmatch(heroicdropsstring, "%[%'%d[^\n^,]+',[^\n^,]-,[^\n^,]-,(%d+),[^\n^,]-,%[[^\n^%]]-b class%=%A%a(%d)[^%]^\n]-%],[^\n^,]-,[^\n^,]-,([^\n^,]-),[^\n^,]-]") do
						droprate = math.floor(tonumber(droprate)*10)
						if tonumber(quality) > 0 and (droprate > 50 or tonumber(quality) > 2) then
							if heroicstr then
								heroicstr = heroicstr..","..itemid..":"..droprate
							else
								heroicstr = itemid..":"..droprate
							end
							count = count + 1
						end
					end
					sets[heroicname] = heroicstr
				else
					local dropsstring = string.match(page, "'Drops',%b[]")
					if dropsstring then
						for itemid, quality, droprate in string.gmatch(dropsstring, "%[%'%d[^\n^,]+',[^\n^,]-,[^\n^,]-,(%d+),[^\n^,]-,%[[^\n^%]]-b class%=%A%a(%d)[^%]^\n]-%],[^\n^,]-,[^\n^,]-,([^\n^,]-),[^\n^,]-]") do
							droprate = math.floor(tonumber(droprate)*10)
							if tonumber(quality) > 0 and (droprate > 50 or tonumber(quality) > 2) then
								if newset then
									newset = newset..","..itemid..":"..droprate
								else
									newset = itemid..":"..droprate
								end
								count = count + 1
							end
						end
					end
				end
				dprint(2, "InstanceLoot: "..boss.." has "..count)
			else
				local page = getpage("http://www.wowhead.com/?objects&filter=na="..url.escape(boss))
				local matchy = (page:match("%'%&filter%=na%=([^%)]+)'%)")):gsub("([%.%(%)%%%+%-%*%?%[%]%^%$])", "%%%1")
				local id = page:match("%['"..matchy.."',[^\n^,]-,(%d+),")
				if id then
					page = getpage("http://www.wowhead.com/?object="..id)
					local count = 0
					local dropsstring = string.match(page, "'Contains',%b[]")
					if dropsstring then
						for itemid, quality, droprate in string.gmatch(dropsstring, "%[%'%d[^\n^,]+',[^\n^,]-,[^\n^,]-,(%d+),[^\n^,]-,%[[^\n^%]]-b class%=%A%a(%d)[^%]^\n]-%],[^\n^,]-,[^\n^,]-,([^\n^,]-),[^\n^,]-]") do
							droprate = math.floor(tonumber(droprate)*10)
							if tonumber(quality) > 0 and (droprate > 50 or tonumber(quality) > 2) then
								if newset then
									newset = newset..","..itemid..":"..droprate
								else
									newset = itemid..":"..droprate
								end
								count = count + 1
							end
						end
					end
					dprint(2, "InstanceLoot: "..boss.." has "..count)
				end
			end
		end
		if set:match("^GearSet") then
			local setname = set:match("%.([^%.]+)$")
			local page = getpage("http://www.wowhead.com/?itemsets&filter=na="..url.escape(setname))
			local matchy = setname:gsub("'", "\'")
			local id = string.match(page, "%['%d["..matchy.."]+[^\n^,]-',[^\n^,]-,[^\n^,]-,(%d+),")
			if id then
				local count = 0
				page = getpage("http://www.wowhead.com/?itemset="..id)
				for itemstring in page:gmatch("(ge%('iconlist%-icon%d+'%)[^\n]+)") do
					local itemid = itemstring:match("'%?item%=(%d+)'")
					if itemid then
						if newset then
							newset = newset..","..itemid
						else
							newset = itemid
						end
						count = count + 1
					else
						itemid = itemstring:match("Item %#(%d+)")
						if itemid then
							if newset then
								newset = newset..","..itemid
							else
								newset = itemid
							end
							count = count + 1
						end
					end
				end
				dprint(2, "GearSet: "..setname.." has "..count)
			end
		end
		if set:match("^Reagent%.Ammo") then
			local setname = set:match("%.([^%.]+)$")
			local count = 0
			if setname == "Arrow" then
				local page = getpage("http://www.wowhead.com/?items&filter=sl=24")
				local itemstring = page:match("'Items',%b[]")
				for itemid, dps in itemstring:gmatch("%[%'%d[^\n^,]+',[^\n^,]-,'Arrow',(%d+),[^\n^,]-,%['[^\n^,]+</table>Adds ([^\n^,]+) damage per second") do
					dps = math.floor(tonumber(dps)*10)
					if newset then
						newset = newset..","..itemid..":"..dps
					else
						newset = itemid..":"..dps
					end
					count = count + 1
				end
			elseif setname == "Bullet" then
				local page = getpage("http://www.wowhead.com/?items&filter=sl=24")
				local itemstring = page:match("'Items',%b[]")
				for itemid, dps in itemstring:gmatch("%[%'%d[^\n^,]+',[^\n^,]-,'Bullet',(%d+),[^\n^,]-,%['[^\n^,]+</table>Adds ([^\n^,]+) damage per second") do
					dps = math.floor(tonumber(dps)*10)
					if newset then
						newset = newset..","..itemid..":"..dps
					else
						newset = itemid..":"..dps
					end
					count = count + 1
				end
			elseif setname == "Thrown" then
				local page = getpage("http://www.wowhead.com/?items&filter=sl=25")
				local itemstring = page:match("'Items',%b[]")
				for itemid, dps in itemstring:gmatch("%[%'%d[^\n^,]+',[^\n^,]-,'Thrown',(%d+),[^\n^,]-,%['[^\n^,]+%(([^\n^,]+) damage per second%)") do
					dps = math.floor(tonumber(dps)*10)
					if newset then
						newset = newset..","..itemid..":"..dps
					else
						newset = itemid..":"..dps
					end
					count = count + 1
				end
			end
			dprint(2, "Reagent.Ammo."..setname..":"..count)
		end
		if set:match("^Bag%.") then
			local setname = set:match("%.([^%.]+)$")
			local count = 0
			local page = getpage("http://www.wowhead.com/?items&filter=sl=18")
			local itemstring = string.match(page, "'Items',%b[]")
			local searchstring
			if setname == "Basic" then
				searchstring = "Bag"
			elseif setname == "Ammo" then
				searchstring = "Ammo Pouch"			
			elseif setname == "Quiver" then
				searchstring = "Quiver"
			elseif setname == "Soul Shard" then
				searchstring = "Soul Bag"
			elseif setname == "Herb" then
				searchstring = "Herb Bag"
			elseif setname == "Enchanting" then
				searchstring = "Enchanting Bag"
			elseif setname == "Engineering" then
				searchstring = "Engineering Bag"
			elseif setname == "Jewelcrafting" then
				searchstring = "Gem Bag"
			elseif setname == "Mining" then
				searchstring = "Mining Bag"
			end
			for itemid, slots in string.gmatch(itemstring, "%[%'%d[^\n^,]+',[^\n^,]-,'"..searchstring.."',(%d+),[^\n^,]-,%['[^\n^,]+%>(%d+) Slot") do
				if newset then
					newset = newset..","..itemid..":"..slots
				else
					newset = itemid..":"..slots
				end
				count = count + 1
			end
		end
		if set:match("^Consumable%.Bandage") then
			local setname = set:match("%.([^%.]+)$")
			local count = 0
			if setname == "Basic" then
				local page = getpage("http://www.wowhead.com/?items&filter=cr=86;crs=6;crv=0")
				local itemstring = string.match(page, "'Items',%b[]")
				for itemid, heal in string.gmatch(itemstring, "%[%'%d[^\n^,]+',[^\n^,]-,'Consumable',(%d+),[^\n^,]-,%['[^\n^,]+Heals (%d+) damage") do
					if newset then
						newset = newset..","..itemid..":"..heal
					else
						newset = itemid..":"..heal
					end
					count = count + 1
				end
			elseif setname == "Alterac Valley" then
				local page = getpage("http://www.wowhead.com/?items&filter=na=bandage;cr=92;crs=1;crv=0")
				local itemstring = string.match(page, "'Items',%b[]")
				for itemid, heal in string.gmatch(itemstring, "%[%'%dAlterac[^\n^,]+',[^\n^,]-,'Consumable',(%d+),[^\n^,]-,%['[^\n^,]+Heals (%d+) damage") do
					if newset then
						newset = newset..","..itemid..":"..heal
					else
						newset = itemid..":"..heal
					end
					count = count + 1
				end
			elseif setname == "Warsong Gulch" then
				local page = getpage("http://www.wowhead.com/?items&filter=na=bandage;cr=92;crs=1;crv=0")
				local itemstring = string.match(page, "'Items',%b[]")
				for itemid, heal in string.gmatch(itemstring, "%[%'%dWarsong[^\n^,]+',[^\n^,]-,'Consumable',(%d+),[^\n^,]-,%['[^\n^,]+Heals (%d+) damage") do
					if newset then
						newset = newset..","..itemid..":"..heal
					else
						newset = itemid..":"..heal
					end
					count = count + 1
				end
			elseif setname == "Arathi Basin" then
				local page = getpage("http://www.wowhead.com/?items&filter=na=bandage;cr=92;crs=1;crv=0")
				local itemstring = string.match(page, "'Items',%b[]")
				for itemid, heal in string.gmatch(itemstring, "%[%'%dDefiler[^\n^,]+',[^\n^,]-,'Consumable',(%d+),[^\n^,]-,%['[^\n^,]+Heals (%d+) damage") do
					if newset then
						newset = newset..","..itemid..":"..heal
					else
						newset = itemid..":"..heal
					end
					count = count + 1
				end
				for itemid, heal in string.gmatch(itemstring, "%[%'%dHighlander[^\n^,]+',[^\n^,]-,'Consumable',(%d+),[^\n^,]-,%['[^\n^,]+Heals (%d+) damage") do
					if newset then
						newset = newset..","..itemid..":"..heal
					else
						newset = itemid..":"..heal
					end
					count = count + 1
				end
			end
		end
		if set:match("^Tradeskill%.Gather%.GemsInNodes") then
			local nodetype = set:match("%.([^%.]+)$")
			local count = 0
			local page
			if nodetype == "Copper Vein" then
				page = getpage("http://www.wowhead.com/?object=1731")
			elseif nodetype == "Tin Vein" then
				page = getpage("http://www.wowhead.com/?object=1732")
			elseif nodetype == "Silver Vein" then
				page = getpage("http://www.wowhead.com/?object=1733")
			elseif nodetype == "Iron Deposit" then
				page = getpage("http://www.wowhead.com/?object=1735")
			elseif nodetype == "Gold Vein" then
				page = getpage("http://www.wowhead.com/?object=1734")
			elseif nodetype == "Mithril Deposit" then
				page = getpage("http://www.wowhead.com/?object=2040")
			elseif nodetype == "Truesilver Deposit" then
				page = getpage("http://www.wowhead.com/?object=2047")
			elseif nodetype == "Small Thorium Vein" then
				page = getpage("http://www.wowhead.com/?object=324")
			elseif nodetype == "Hakkari Thorium Vein" then
				page = getpage("http://www.wowhead.com/?object=180215")
			elseif nodetype == "Rich Thorium Vein" then
				page = getpage("http://www.wowhead.com/?object=175404")
			elseif nodetype == "Dark Iron Deposit" then
				page = getpage("http://www.wowhead.com/?object=165658")
			elseif nodetype == "Fel Iron Deposit" then
				page = getpage("http://www.wowhead.com/?object=181555")
			elseif nodetype == "Adamantite Deposit" then
				page = getpage("http://www.wowhead.com/?object=181556")
			elseif nodetype == "Rich Adamantite Deposit" then
				page = getpage("http://www.wowhead.com/?object=181569")
			elseif nodetype == "Khorium Vein" then
				page = getpage("http://www.wowhead.com/?object=181557")
			end
			local itemstring = page:match("'Mining',%b[]")
			for itemid in string.gmatch(itemstring, "%[%'%d[^\n^,]+',[^\n^,]-,'Raw Gem',(%d+),") do
				if newset then
					newset = newset..","..itemid
				else
					newset = itemid
				end
				count = count + 1
			end
		end
		if set:match("^Tradeskill%.Crafted") then
			local count = 0
			local profession = set:match("^Tradeskill%.Crafted%.(.+)$")
			local page
			if profession == "Alchemy" then
				page = getpage("http://www.wowhead.com/?spells=11.171")
			elseif profession == "Blacksmithing.Basic" then
				page = getpage("http://www.wowhead.com/?spells=11.164&filter=cr=5;crs=2;crv=0")
			elseif profession == "Blacksmithing.Armorsmith" then
				page = getpage("http://www.wowhead.com/?spells=11.164.9788")
			elseif profession == "Blacksmithing.Weaponsmith.Axesmith" then
				page = getpage("http://www.wowhead.com/?spells=11.164.17041")
			elseif profession == "Blacksmithing.Weaponsmith.Hammersmith" then
				page = getpage("http://www.wowhead.com/?spells=11.164.17040")
			elseif profession == "Blacksmithing.Weaponsmith.Swordsmith" then
				page = getpage("http://www.wowhead.com/?spells=11.164.17039")
			elseif profession == "Blacksmithing.Weaponsmith.Basic" then
				page = getpage("http://www.wowhead.com/?spells=11.164.9787")
			elseif profession == "Cooking" then
				page = getpage("http://www.wowhead.com/?spells=9.185")
			elseif profession == "Enchanting" then
				page = getpage("http://www.wowhead.com/?spells=11.333")
			elseif profession == "Engineering.Basic" then
				page = getpage("http://www.wowhead.com/?spells=11.202&filter=cr=5;crs=2;crv=0")
			elseif profession == "Engineering.Gnomish" then
				page = getpage("http://www.wowhead.com/?spells=11.202.20219")
			elseif profession == "Engineering.Goblin" then
				page = getpage("http://www.wowhead.com/?spells=11.202.20222")
			elseif profession == "First Aid" then
				page = getpage("http://www.wowhead.com/?spells=9.129")
			elseif profession == "Jewelcrafting" then
				page = getpage("http://www.wowhead.com/?spells=11.755")
			elseif profession == "Leatherworking.Basic" then
				page = getpage("http://www.wowhead.com/?spells=11.165&filter=cr=5;crs=2;crv=0")
			elseif profession == "Leatherworking.Dragonscale" then
				page = getpage("http://www.wowhead.com/?spells=11.165.10656")
			elseif profession == "Leatherworking.Elemental" then
				page = getpage("http://www.wowhead.com/?spells=11.165.10658")
			elseif profession == "Leatherworking.Tribal" then
				page = getpage("http://www.wowhead.com/?spells=11.165.10660")
			elseif profession == "Smelting" then
				page = getpage("http://www.wowhead.com/?spells=11.186")
			elseif profession == "Tailoring.Basic" then
				page = getpage("http://www.wowhead.com/?spells=11.197&filter=cr=5;crs=2;crv=0")
			elseif profession == "Tailoring.Mooncloth" then
				page = getpage("http://www.wowhead.com/?spells=11.197.26798")
			elseif profession == "Tailoring.Shadoweave" then
				page = getpage("http://www.wowhead.com/?spells=11.197.26801")
			elseif profession == "Tailoring.Spellfire" then
				page = getpage("http://www.wowhead.com/?spells=11.197.26797")
			end
			if page then
				sets["TradeskillResultMats.Forward."..profession] = nil
				sets["TradeskillResultMats.Reverse."..profession] = "o"
				sets["Tradeskill.RecipeLinks."..profession] = nil
				local reagenttable = {}
				local itemstring = page:match("'Spells',%b[]")
				if not itemstring then print(page.."\n\n\n"..itemstring.."\n\n\n".."ERR"..profession..set) return end
				for spellid, skilllvl, reagentstring in string.gmatch(itemstring, "%[%'.[^\n^,]+',[^\n^,]-,[^\n^,]-,(%d+),[^\n^,]-,%b[],(%d+),(%b[])") do
					if tonumber(skilllvl) == 99999 then
						skilllvl = 375
					end
					local itemid
					if profession == "Enchanting" then
						itemid = -1 * spellid
						if itemid and tonumber(skilllvl) > 0 and tonumber(skilllvl) <=375 and reagentstring ~= "[]" then
							if newset then
								newset = newset..","..itemid..":"..skilllvl
							else
								newset = itemid..":"..skilllvl
							end
							local newrecipemats = itemid..":"
							dprint(3, reagentstring)
							for reagentid,reagentnum in reagentstring:sub(2,-2):gmatch("%[(%d+),(%d+)%]") do
								if reagenttable[reagentid] then
									reagenttable[reagentid] = reagenttable[reagentid]..";"..itemid.."x"..reagentnum
								else
									reagenttable[reagentid] = itemid.."x"..reagentnum
								end
								dprint(3, reagentid)
								newrecipemats = newrecipemats..reagentid.."x"..reagentnum..";"
							end
							newrecipemats = newrecipemats:sub(1,-2)
							if sets["TradeskillResultMats.Forward."..profession] then
								sets["TradeskillResultMats.Forward."..profession] = sets["TradeskillResultMats.Forward."..profession]..","..newrecipemats
							else
								sets["TradeskillResultMats.Forward."..profession] = "o,"..newrecipemats
							end
							count = count + 1
							dprint(3, profession..skilllvl..":"..itemid.." -- "..newrecipemats)
						end
					else
						local subpage = getpage("http://www.wowhead.com/?spell="..spellid)
						if not subpage then
							dprint(1, " -> check page http://www.wowhead.com/?spell="..spellid)
							subpage = ""
						end
						itemid = string.match(subpage, "Create Item.-?item=(%d+)")
						if itemid and tonumber(skilllvl) > 0 and tonumber(skilllvl) <=375 then
							if newset then
								newset = newset..","..itemid..":"..skilllvl
							else
								newset = itemid..":"..skilllvl
							end
							local newrecipemats = itemid..":"
							dprint(3, reagentstring)
							for reagentid,reagentnum in reagentstring:sub(2,-2):gmatch("%[(%d+),(%d+)%]") do
								if reagenttable[reagentid] then
									reagenttable[reagentid] = reagenttable[reagentid]..";"..itemid.."x"..reagentnum
								else
									reagenttable[reagentid] = itemid.."x"..reagentnum
								end
								dprint(3, reagentid)
								newrecipemats = newrecipemats..reagentid.."x"..reagentnum..";"
							end
							newrecipemats = newrecipemats:sub(1,-2)
							if sets["TradeskillResultMats.Forward."..profession] then
								sets["TradeskillResultMats.Forward."..profession] = sets["TradeskillResultMats.Forward."..profession]..","..newrecipemats
							else
								sets["TradeskillResultMats.Forward."..profession] = "o,"..newrecipemats
							end
							count = count + 1
							dprint(3, profession..skilllvl..":"..itemid.." -- "..newrecipemats)
						end
					end
					if itemid then
						if tonumber(skilllvl) > 0 and tonumber(skilllvl) <=375 then
							if sets["Tradeskill.RecipeLinks."..profession] then
								sets["Tradeskill.RecipeLinks."..profession] = sets["Tradeskill.RecipeLinks."..profession]..",-"..spellid..":"..itemid
							else
								sets["Tradeskill.RecipeLinks."..profession] = "-"..spellid..":"..itemid
							end
						end
					end
				end
				
				for k,v in pairs(reagenttable) do
					sets["TradeskillResultMats.Reverse."..profession] = sets["TradeskillResultMats.Reverse."..profession]..","..k..":"..v
				end
			end
		end
		if set:match("^Tradeskill%.Gather") then
			local count = 0
			local gathertype = set:match("%.([^%.]+)$")
			local page
			if gathertype == "Fishing" then
				page = getpage("http://www.wowhead.com/?items&filter=cr=69;crs=1;crv=0")
			elseif gathertype == "Disenchant" then
				page = getpage("http://www.wowhead.com/?items&filter=cr=68;crs=1;crv=0")
			elseif gathertype == "Herbalism" then
				page = getpage("http://www.wowhead.com/?items&filter=cr=70;crs=1;crv=0")
			elseif gathertype == "Mining" then
				page = getpage("http://www.wowhead.com/?items&filter=cr=73;crs=1;crv=0")
			elseif gathertype == "Skinning" then
				page = getpage("http://www.wowhead.com/?items&filter=cr=76;crs=1;crv=0")
			elseif gathertype == "Prospecting" then
				page = getpage("http://www.wowhead.com/?items&filter=cr=88;crs=1;crv=0")
			elseif gathertype == "Pickpocketing" then
				page = getpage("http://www.wowhead.com/?items&filter=cr=75;crs=1;crv=0")
			end
			if page then
				local itemstring = string.match(page, "'Items',%b[]")
				for itemid in string.gmatch(itemstring, "%[%'%d[^\n^,]+',[^\n^,]-,[^\n^,]-,(%d+),") do
					if newset then
						newset = newset..","..itemid
					else
						newset = itemid
					end
					count = count + 1
				end
			end
		end
		if set:match("^Tradeskill%.Recipe%.") then
			local count = 0
			local profession, filter = set:match("^Tradeskill%.Recipe%.([^%.]+)%.(.+)$")
			local url
			if profession == "Alchemy" then
				url = "http://www.wowhead.com/?items=9.6"
			elseif profession == "Blacksmithing" then
				url = "http://www.wowhead.com/?items=9.4"
			elseif profession == "Cooking" then
				url = "http://www.wowhead.com/?items=9.5"
			elseif profession == "Enchanting" then
				url = "http://www.wowhead.com/?items=9.8"
			elseif profession == "Engineering" then
				url = "http://www.wowhead.com/?items=9.3"
			elseif profession == "First Aid" then
				url = "http://www.wowhead.com/?items=9.7"
			elseif profession == "Fishing" then
				url = "http://www.wowhead.com/?items=9.9"
			elseif profession == "Jewelcrafting" then
				url = "http://www.wowhead.com/?items=9.10"
			elseif profession == "Leatherworking" then
				url = "http://www.wowhead.com/?items=9.1"
			elseif profession == "Tailoring" then
				url = "http://www.wowhead.com/?items=9.2"
			end
			
			if filter == "Crafted" then
				url = url and (url.."&filter=cr=86;crs=11;crv=0") or nil
			elseif filter == "Drop" then
				url = url and (url.."&filter=cr=72;crs=1;crv=0") or nil
			elseif filter == "Other" then
				url = url and (url.."&filter=cr=18,72,86,92;crs=5,2,12,2;crv=0,0,0,0") or nil
			elseif filter == "Quest" then
				url = url and (url.."&filter=cr=18;crs=1;crv=0") or nil
			elseif filter == "Vendor" then
				url = url and (url.."&filter=cr=92;crs=1;crv=0") or nil
			else
				url = nil
			end
			if url then
				page = getpage(url)
			end
			
			if page then
				local itemstring = string.match(page, "'Items',%b[]")
				if not itemstring then
					if not string.match(page, "Your criteria did not match any items%.") then
						--error("ALERT PAGE DIDNT LOAD "..profession.."."..filter)
					end
					itemstring = ""
				end
				for skilllvl, itemid in string.gmatch(itemstring, "%[%'%d[^\n^,]+',[^\n^,]-,([^\n^,]-),[^\n^,]-,(%d+),") do
					if newset then
						newset = newset..","..itemid..":"..skilllvl
					else
						newset = itemid..":"..skilllvl
					end
					count = count + 1
				end
			end
		end
		if set:match("^Tradeskill%.Tool") then
			local count = 0
			local profession = set:match("^Tradeskill%.Tool%.(.+)$")
			local urls = {}
			local page
			if profession == "Alchemy" then
				table.insert(urls, "http://www.wowhead.com/?items&filter=cr=91;crs=12;crv=0") -- Tool - Philosopher's Stone
			elseif profession == "Blacksmithing" then
				table.insert(urls, "http://www.wowhead.com/?items&filter=cr=91;crs=13;crv=0") -- Tool - Blacksmith Hammer
			elseif profession == "Enchanting" then
				table.insert(urls, "http://www.wowhead.com/?items&filter=cr=91;crs=62;crv=0") -- Tool - Runed Adamantite Rod
				table.insert(urls, "http://www.wowhead.com/?items&filter=cr=91;crs=10;crv=0") -- Tool - Runed Arcanite Rod
				table.insert(urls, "http://www.wowhead.com/?items&filter=cr=91;crs=6;crv=0")  -- Tool - Runed Copper Rod
				table.insert(urls, "http://www.wowhead.com/?items&filter=cr=91;crs=63;crv=0") -- Tool - Runed Eternium Rod
				table.insert(urls, "http://www.wowhead.com/?items&filter=cr=91;crs=41;crv=0") -- Tool - Runed Fel Iron Rod
				table.insert(urls, "http://www.wowhead.com/?items&filter=cr=91;crs=8;crv=0")  -- Tool - Runed Golden Rod
				table.insert(urls, "http://www.wowhead.com/?items&filter=cr=91;crs=7;crv=0")  -- Tool - Runed Silver Rod
				table.insert(urls, "http://www.wowhead.com/?items&filter=cr=91;crs=9;crv=0")  -- Tool - Runed Truesilver Rod
			elseif profession == "Engineering" then
				table.insert(urls, "http://www.wowhead.com/?items&filter=cr=91;crs=14;crv=0") -- Tool - Arclight Spanner
				table.insert(urls, "http://www.wowhead.com/?items&filter=cr=91;crs=13;crv=0") -- Tool - Blacksmith Hammer
				table.insert(urls, "http://www.wowhead.com/?items&filter=cr=91;crs=15;crv=0") -- Tool - Gyromatic Micro-Adjustor
--			elseif profession == "Jewelcrafting" then 
			elseif profession == "Mining" then
				table.insert(urls, "http://www.wowhead.com/?items&filter=cr=91;crs=11;crv=0") -- Tool - Mining Pick
			elseif profession == "Skinning" then
				table.insert(urls, "http://www.wowhead.com/?items&filter=cr=91;crs=1;crv=0")  -- Tool - Skinning Knife
			end
			for _, url in ipairs(urls) do
				page = getpage(url)
				if page then
					local itemstring = string.match(page, "'Items',%b[]")
					if itemstring then
						for itemid in string.gmatch(itemstring, "%[%'%d[^\n^,]+',[^\n^,]-,[^\n^,]-,(%d+),") do
							if newset then
								newset = newset..","..itemid
							else
								newset = itemid
							end
							count = count + 1
						end
					end
				end
			end
		end
		if set:match("^Tradeskill%.Mat%.ByProfession") then
			-- print(set)
			local count = 0
			local profession = set:match("^Tradeskill%.Mat%.ByProfession%.(.+)$")
			local page
			if profession == "Alchemy" then
				page = getpage("http://www.wowhead.com/?spells=11.171")
			elseif profession == "Blacksmithing.Basic" then
				page = getpage("http://www.wowhead.com/?spells=11.164&filter=cr=5;crs=2;crv=0")
			elseif profession == "Blacksmithing.Armorsmith" then
				page = getpage("http://www.wowhead.com/?spells=11.164.9788")
			elseif profession == "Blacksmithing.Weaponsmith.Axesmith" then
				page = getpage("http://www.wowhead.com/?spells=11.164.17041")
			elseif profession == "Blacksmithing.Weaponsmith.Hammersmith" then
				page = getpage("http://www.wowhead.com/?spells=11.164.17040")
			elseif profession == "Blacksmithing.Weaponsmith.Swordsmith" then
				page = getpage("http://www.wowhead.com/?spells=11.164.17039")
			elseif profession == "Blacksmithing.Weaponsmith.Basic" then
				page = getpage("http://www.wowhead.com/?spells=11.164.9787")
			elseif profession == "Cooking" then
				page = getpage("http://www.wowhead.com/?spells=9.185")
			elseif profession == "Enchanting" then
				page = getpage("http://www.wowhead.com/?spells=11.333")
			elseif profession == "Engineering.Basic" then
				page = getpage("http://www.wowhead.com/?spells=11.202&filter=cr=5;crs=2;crv=0")
			elseif profession == "Engineering.Gnomish" then
				page = getpage("http://www.wowhead.com/?spells=11.202.20219")
			elseif profession == "Engineering.Goblin" then
				page = getpage("http://www.wowhead.com/?spells=11.202.20222")
			elseif profession == "First Aid" then
				page = getpage("http://www.wowhead.com/?spells=9.129")
			elseif profession == "Jewelcrafting" then
				page = getpage("http://www.wowhead.com/?spells=11.755")
			elseif profession == "Leatherworking.Basic" then
				page = getpage("http://www.wowhead.com/?spells=11.165&filter=cr=5;crs=2;crv=0")
			elseif profession == "Leatherworking.Dragonscale" then
				page = getpage("http://www.wowhead.com/?spells=11.165.10656")
			elseif profession == "Leatherworking.Elemental" then
				page = getpage("http://www.wowhead.com/?spells=11.165.10658")
			elseif profession == "Leatherworking.Tribal" then
				page = getpage("http://www.wowhead.com/?spells=11.165.10660")
			elseif profession == "Smelting" then
				page = getpage("http://www.wowhead.com/?spells=11.186")
			elseif profession == "Tailoring.Basic" then
				page = getpage("http://www.wowhead.com/?spells=11.197&filter=cr=5;crs=2;crv=0")
			elseif profession == "Tailoring.Mooncloth" then
				page = getpage("http://www.wowhead.com/?spells=11.197.26798")
			elseif profession == "Tailoring.Shadoweave" then
				page = getpage("http://www.wowhead.com/?spells=11.197.26801")
			elseif profession == "Tailoring.Spellfire" then
				page = getpage("http://www.wowhead.com/?spells=11.197.26797")
			end
			if page then
				for itemid in page:gmatch("%_%[(%d+)%]") do
					if not newset or (newset and not newset:match(itemid)) then
						if newset then
							newset = newset..","..itemid
						else
							newset = itemid
						end
						count = count + 1
					end
				end
			end
		end
		if set:match("^Tradeskill%.Gem") and not set:match("^Tradeskill%.Gem%.Cut") then
			local color = set:match("%.([^%.]+)$")
			local count = 0
			local colorpages = {
				["Red"] = "http://www.wowhead.com/?items=3.0",
				["Blue"] = "http://www.wowhead.com/?items=3.1",
				["Yellow"] = "http://www.wowhead.com/?items=3.2",
				["Green"] = "http://www.wowhead.com/?items=3.4",
				["Purple"] = "http://www.wowhead.com/?items=3.3",
				["Orange"] = "http://www.wowhead.com/?items=3.5",
				["Meta"] = "http://www.wowhead.com/?items=3.6",
				["Prismatic"] = "http://www.wowhead.com/?items=3.8",
			}
			local page = getpage(colorpages[color])
			local itemstring = page:match("'Items',%b[]")
			for itemid, types in itemstring:gmatch("%[[^,]-,[^,]-,[^,]-,(%d+),[^,]-,%b[]") do
				if newset then
					newset = newset..","..itemid
				else
					newset = itemid
				end
				count = count + 1
			end
		end
		if set:match("^Tradeskill%.Gem%.Cut") then
			local gems = {
				["23112"] = "",
				["23107"] = "",
				["21929"] = "",
				["23079"] = "",
				["23117"] = "",
				["23077"] = "",
				["23436"] = "",
				["23440"] = "",
				["23437"] = "",
				["23439"] = "",
				["23441"] = "",
				["23438"] = "",
				["25867"] = "",
				["25868"] = "",
			}
			local count = 0
			local page = getpage("http://www.wowhead.com/?items&filter=cr=81;crs=5;crv=0")
			local itemstring = page:match("'Items',%b[]")
			for itemid in itemstring:gmatch("%[[^,]-,[^,]-,[^,]-,(%d+),[^,]-,%b[]") do
				page = getpage("http://www.wowhead.com/?item="..itemid)
				local reagent = page:match("_%[(%d+)%]")
				dprint(3, itemid)
				dprint(3, reagent)
				if reagent and gems[reagent] then
					gems[reagent] = gems[reagent]..";"..itemid
				end
			end
			for k,v in pairs(gems) do
				if newset then
					newset = newset..","..k..":"..v:sub(2)
				else
					newset = "o,"..k..":"..v:sub(2)
				end
			end
		end
		if set:match("^Misc%.Minipet%.Normal") then
			local count = 0
			local page = getpage("http://www.wowhead.com/?items=15.-2")
			local itemstring = page:match("'Items',%b[]")
			for itemid in itemstring:gmatch("%[%'%d[^\n^,]+',[^\n^,]-,'Small Pet',(%d+),[^\n^,]-,%b[]") do
				if newset then
					newset = newset..","..itemid
				else
					newset = itemid
				end
				count = count + 1
			end
		end
		if set:match("^Misc%.Lockboxes") then
			local count = 0
			local page = getpage("http://www.wowhead.com/?items&filter=cr=10;crs=1;crv=0")
			local itemstring = page:match("'Items',%b[]")
			for itemid in itemstring:gmatch("%[%'%d[^\n^,]+',[^\n^,]-,'Junk',(%d+),[^\n^,]-,%b[]") do
				if newset then
					newset = newset..","..itemid
				else
					newset = itemid
				end
				count = count + 1
			end
		end
		if set:match("^Gear%.Socketed") and not set:match("^Tradeskill%.Gem%.Cut") then
			local slot = set:match("%.([^%.]+)$")
			local count = 0
			local slotpages = {
				["Back"]	= {"http://www.wowhead.com/?items&filter=sl=16;cr=80;crs=5;crv=0"},
				["Chest"]	= {"http://www.wowhead.com/?items&filter=sl=5;cr=72,80;crs=1,5;crv=0,0",
						"http://www.wowhead.com/?items&filter=sl=5;cr=93,80;crs=1,5;crv=0,0",
						"http://www.wowhead.com/?items&filter=sl=5;cr=93,80,72;crs=2,5,2;crv=0,0,0"},
				["Feet"]	= {"http://www.wowhead.com/?items&filter=sl=8;cr=80;crs=5;crv=0"},
				["Finger"]	= {"http://www.wowhead.com/?items&filter=sl=11;cr=80;crs=5;crv=0"},
				["Hands"]	= {"http://www.wowhead.com/?items&filter=sl=10;cr=80;crs=5;crv=0"},
				["Head"]	= {"http://www.wowhead.com/?items&filter=sl=1;cr=80,72;crs=5,1;crv=0,0",
						"http://www.wowhead.com/?items&filter=sl=1;cr=80,93;crs=5,1;crv=0,0",
						"http://www.wowhead.com/?items&filter=sl=1;cr=80,93,72;crs=5,2,2;crv=0,0,0"},
				["Legs"]	= {"http://www.wowhead.com/?items&filter=sl=7;cr=80;crs=5;crv=0"},
				["Main Hand"]	= {"http://www.wowhead.com/?items&filter=sl=21;cr=80;crs=5;crv=0"},
				["Neck"]	= {"http://www.wowhead.com/?items&filter=sl=2;cr=80;crs=5;crv=0"},
				["One Hand"]	= {"http://www.wowhead.com/?items&filter=sl=13;cr=80;crs=5;crv=0"},
				["Ranged"]	= {"http://www.wowhead.com/?items&filter=sl=15;cr=80;crs=5;crv=0"},
				["Shield"]	= {"http://www.wowhead.com/?items&filter=sl=14;cr=80;crs=5;crv=0"},
				["Shoulder"]	= {"http://www.wowhead.com/?items&filter=sl=3;cr=80,72;crs=5,1;crv=0,0",
						"http://www.wowhead.com/?items&filter=sl=3;cr=80,93;crs=5,1;crv=0,0",
						"http://www.wowhead.com/?items&filter=sl=3;cr=80,93,72;crs=5,2,2;crv=0,0,0"},
				["Two Hand"]	= {"http://www.wowhead.com/?items&filter=sl=17;cr=80;crs=5;crv=0"},
				["Waist"]	= {"http://www.wowhead.com/?items&filter=sl=6;cr=80;crs=5;crv=0"},
				["Wrist"]	= {"http://www.wowhead.com/?items&filter=sl=9;cr=80;crs=5;crv=0"},
			}
			for _,url in ipairs(slotpages[slot]) do
				local page = getpage(url)
				local itemstring = page:match("'Items',%b[]")
				for itemid, types in itemstring:gmatch("%[[^,]-,[^,]-,[^,]-,(%d+),[^,]-,%b[]") do
					if newset then
						newset = newset..","..itemid
					else
						newset = itemid
					end
					count = count + 1
				end
			end
		end
		if set:match("Container%.ItemsInType") then
			local containers = {
				["Soul Shard"] = 21342,
				Herb = 22251,
				Enchanting = 21858,
				Engineering = 30745,
				Gem = 30747,
				Mining = 29540,
			}
			local container = set:match("%.([^%.]+)$")
			local page = getpage("http://www.wowhead.com/?item="..(containers[container] or ''))
			local itemstring = page:match("'Can contain',%b[]")
			for itemid in itemstring:gmatch("%[[^,]-,[^,]-,[^,]-,(%d+),[^,]-,%b[]") do
				if newset then
					newset = newset..","..itemid
				else
					newset = itemid
				end
			end
		end
	end
	if newset then
		sets[set] = newset
	else
		table.insert(notmined,set)
	end
end
local elapsed = os.time()- starttime
print(("Elapsed Time: %dm %ds"):format(elapsed/60, elapsed%60))
print(("%dm %ds spent in web requests"):format(httptime/60, httptime%60))
print(("%dm %ds spent in processing data"):format((elapsed-httptime)/60,(elapsed-httptime)%60))
print(("Approx %dm %.2fs CPU time used"):format(os.clock()/60, os.clock()%60))
local notminedcount = 0
for k,v in ipairs(notmined) do
	--print("not mined:"..v)
	notminedcount = notminedcount + 1
end
print(("%d sets mined, %d sets not mined."):format(setcount-notminedcount, notminedcount))
local f = assert(io.open("data.lua", "w"))
for line in file:gmatch('([^\n]-\n)') do
	local setname, spaces, comment = line:match('\t%[%"([^"]+)%"%]([^=]-)= "[^"]-",([^\n]-)\n')
	if setname and sets[setname] then
		f:write('\t["'..setname..'"]'..spaces..'= "'..sets[setname]..'",'..comment..'\n')
	else
		f:write(line)
	end
end
f:close()
