관세청홈페이지 크롤링 이용하기

관세청에서 확인할 내용

개발자 모드 Network에서 retrieveTradeRegion.do 확인
Header에서 general, header등 확인.

중요한건 payload : 검색날짜, 시도 코드, HS코드 길이, HS코드 등 중요

파이썬 코드 작성

def bhi_trade(self): #비에이치아이
    try:
        check_open_file = open("D:\\4.각종INDEX\\★기업별수출입★.xlsx", "+r")
        check_open_file.close()
    except:
        QMessageBox.about(self, "message", "엑셀 close!!")
        return 0
    strdate = self.dateEdit_str.date() #.toString("yyyyMM")
    enddate = self.dateEdit_end.date() #.toString("yyyyMM")

    sido = ['48']  # 경상남도
    sgg =  ['637'] # 함안
    hscode = ['84']  # 라면
    self.ret_value = self.tradeWeb.get_tradeDataWeb(strdate, enddate, sido,sgg,hscode)

    self.error_message("비에이치아이", self.ret_value)
    pass

class TradedataWebPage():
    def __init__(self):
        self.url = 'http://apis.data.go.kr/1220000/sidoitemtrade/getSidoitemtradeList'
        self.strYYmm = {}
        self.endYYmm = {}
        self.sido = {}
        self.sgg = {}
        self.strHS_NumOfDigit = {}
        self.hashcode = {}
        self.url_tradedata_go_kr = 'https://tradedata.go.kr/cts/hmpg/retrieveTradeRegion.do'  # 지역별-품목별-수출입실적(시군구)
        self.headers = {  # 만질필요없는듯
            'Accept': 'application/json, text/javascript, */*; q=0.01',
            'Accept-Encoding': 'gzip, deflate, br',
            'Accept-Language': 'ko-KR,ko;q=0.9,en-US;q=0.8,en;q=0.7',
            'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
            'Host': 'tradedata.go.kr',
            'isAjax': 'true',
            'Origin': 'https://tradedata.go.kr',
            'Referer': 'https://tradedata.go.kr/cts/index.do',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.104 Whale/3.13.131.36 Safari/537.36',
            'X-Requested-With': 'XMLHttpRequest',
        }
        self.params = {
            'tradeKind': 'ETS_MNK_1040000A',
            'sidosggKind': 'sgg',
            'priodKind': 'MON',
            'priodFr': f'{self.strYYmm}',
            'priodTo': f'{self.endYYmm}',
            'statsBase': 'acptDd',
            'sidoCd': f'{self.sido}', #도
            'sggCd': f'{self.sgg}', #시군군구
            'imexTpcd': 'EXP_TMPR_CD',
            'showPagingLine': '15',
            'sortColumn': '',
            'sortOrder': '',
            'hsSgnGrpCol': f'{self.strHS_NumOfDigit}', #HS4_SGN HS코드자리수 ex) 19(곡물),1902(파스타),190230(그밖의파스타) //6자리까지
            'hsSgnWhrCol': f'{self.strHS_NumOfDigit}',
            'hsSgn': f'{self.hashcode}',  #
        }

def get_tradeDataWeb(self,strdate,enddate,sido,sgg,hscode):
    strdate = strdate.toString("yyyyMM")
    enddate = enddate.toString("yyyyMM")

    self.df_tradeData = []
    df_hs = pd.DataFrame()

    for hs in range(len(hscode)):
        hsnum = 'HS' + str(len(hscode[hs])) + '_SGN'  # HS4_SGN, HS6_SGN
        df_sido = pd.DataFrame()
        for si in range(len(sido)):
            self.params.update({'priodFr':strdate, 'priodTo':enddate, 'sidoCd':sido[si], 'sggCd':sgg[si],'hsSgnGrpCol':hsnum,'hsSgnWhrCol':hsnum,'hsSgn':hscode[hs]})
            self.df_tradeData = self.sub_get_tradeDataWeb()
            df_sido = df_sido._append(self.df_tradeData)
        df_hs = df_hs._append(df_sido)

def sub_get_tradeDataWeb(self):
    res = requests.post(self.url_tradedata_go_kr, headers=self.headers, params=self.params)
    data = json.loads(res.text)