You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
46 KiB
46 KiB
페키지 로드¶
In [1]:
%reload_ext watermark %watermark -v -p numpy,matplotlib,pandas,sklearn,tqdm,tensorflow,rpy2,watermark
CPython 3.6.9 IPython 7.16.2 numpy 1.19.5 matplotlib 3.3.4 pandas 1.1.5 sklearn 0.24.2 tqdm 4.62.3 tensorflow 2.6.2 rpy2 3.4.5 watermark 2.0.2
In [2]:
from rpy2.robjects import pandas2ri from rpy2.robjects import r from rpy2.robjects.packages import importr pandas2ri.activate() utils = importr('utils') package_names = ('ranger') utils.chooseCRANmirror(ind=1) #utils.install_packages("ranger") # ranger 패키지 설치
Out[2]:
<rpy2.rinterface_lib.sexp.NULLType object at 0x7fd294ca7288> [RTYPES.NILSXP]
In [3]:
import numpy as np import pandas as pd import os, re, cv2 from tqdm.auto import tqdm import matplotlib.pyplot as plt os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"]="0" main_dir="/root/data/dacon/open"
함수 생성¶
In [4]:
# 색 강조 def img_Contrast(img,clipLimit=3.0,tileGridSize=(8,8)): lab=cv2.cvtColor(img, cv2.COLOR_BGR2LAB) l,a,b=cv2.split(lab) clahe=cv2.createCLAHE(clipLimit=clipLimit, tileGridSize=tileGridSize) cl=clahe.apply(l) limg=cv2.merge((cl,a,b)) final = cv2.cvtColor(limg,cv2.COLOR_LAB2BGR) return final # 특정범위 색 추출 def img_extract(return_img,img,lower=(0,0,0), upper=(110,255,200)): img_hsv=cv2.cvtColor(img, cv2.COLOR_BGR2HSV) img_mask=cv2.inRange(img_hsv, lower, upper) img_result= cv2.bitwise_and(return_img, return_img, mask=img_mask) img_hsv=cv2.cvtColor(img, cv2.COLOR_HSV2RGB) return img_result # 파일 목록 def file_list(directory): def find_files(directory): return([f"{directory}/{i}" \ for i in os.listdir(directory) if re.compile('png$|jpg$').findall(i)]) out=list() if type(directory)==str: out=find_files(directory) elif type(directory)==list: for folder in range(len(directory)): [out.append(file) for file in find_files(directory[folder])] return( sorted(out)) # 이미지 통계량 추출 def rgb_stat(img): r_m,g_m,b_m =np.mean(img,axis=(0,1)) r_sd,g_sd,b_sd= np.std(img,axis=(0,1)) return r_m,g_m,b_m,r_sd,g_sd,b_sd # 무게 산출 def img_to_weight(img,n=15000): return (cv2.cvtColor(img,cv2.COLOR_RGB2GRAY)!=0).sum()/n # 파일 목록 def file_list(directory): def find_files(directory): return([f"{directory}/{i}" \ for i in os.listdir(directory) if re.compile('png$|jpg$').findall(i)]) out=list() if type(directory)==str: out=find_files(directory) elif type(directory)==list: for folder in range(len(directory)): [out.append(file) for file in find_files(directory[folder])] return( sorted(out)) # 이미지 변수 생성 def img_feature(dirs): df=pd.DataFrame({'img_dirs':dirs}) for i in tqdm(range(len(dirs))): img=cv2.imread(dirs[i]) raw_img=cv2.resize(img,dsize=(int(img.shape[0]/4),int(img.shape[1]/4)),interpolation=cv2.INTER_CUBIC) contrast_img=img_Contrast(img, 3, tileGridSize=(4,3)) # 시든 잎 추출 extract_img=img_extract(img,contrast_img, lower=(10,0,0), upper=(30,255,255)) df.loc[i,'del_leaf']=img_to_weight(extract_img,16900) # 청경채 추출 extract_img=img_extract(img,contrast_img,upper=(90,255,130))# 원본에서 추출 df.loc[i,'pred_leaf1']=img_to_weight(extract_img,16900) contrast_img=img_Contrast(img, 3, tileGridSize=(5,5)) extract_img=img_extract(img,contrast_img,upper=(77,255,130))# 원본에서 추출 df.loc[i,'pred_leaf2']=img_to_weight(extract_img,16900) df.loc[i,'pred_leaf_mean']=(df.loc[i,'pred_leaf1']+df.loc[i,'pred_leaf2'])/2 #RGB 추출 df.loc[i,["r_m","g_m","b_m","r_sd","g_sd","b_sd"]]=rgb_stat(extract_img) return df
자료생성¶
In [5]:
tr_directory=[f"{main_dir}/train/CASE{i:02d}/image" for i in range(1,76)] tr_img_dirs=file_list(tr_directory) tr_img_dirs.remove(f'{main_dir}/train/CASE45/image/CASE45_17.png') te_img_dirs=file_list(f"{main_dir}/test/image") label_dfs=list() for file in tqdm([f"{main_dir}/train/CASE{i:02d}/label.csv" for i in range(1,76)]): temp_df=pd.read_csv(file) for i, img_file in enumerate(temp_df.img_name): time_df=pd.read_csv( f"{main_dir}/train/CASE{img_file[4:6]}/meta/{img_file.replace('jpg','png').replace('png','csv')}") time_df=time_df.sort_values('시간') time=time_df.loc[0,'시간'] temp_df.loc[i,'date']=pd.to_datetime(time).date() label_dfs.append(temp_df) label_df=pd.concat(label_dfs) label_df['case']=[i[:6] for i in label_df['img_name']] merge_df=label_df.copy() merge_df.columns=['img_name','now_weight','date','case'] merge_df=merge_df.drop("img_name",axis=1) merge_df['date']=merge_df.date+pd.to_timedelta(1,unit='day') label_df=pd.merge(label_df,merge_df,how='left',on=['case','date']) del merge_df
0%| | 0/75 [00:00<?, ?it/s]
In [6]:
if not('tr_df.csv' in os.listdir("/root/jupyter/데이콘/청경채/input/")): tr_df=img_feature(tr_img_dirs) te_df=img_feature(te_img_dirs) tr_df.to_csv('/root/jupyter/데이콘/청경채/input/tr_df.csv',index=False) te_df.to_csv('/root/jupyter/데이콘/청경채/input/te_df.csv',index=False) else: tr_df=pd.read_csv('/root/jupyter/데이콘/청경채/input/tr_df.csv') te_df=pd.read_csv('/root/jupyter/데이콘/청경채/input/te_df.csv') #라벨 for i in tqdm(range(tr_df.shape[0])): tr_df.loc[i,['leaf_weight','date','now_weight']]=label_df.loc[ label_df.img_name==tr_df.img_dirs[i].split('/')[-1],['leaf_weight','date','now_weight']].values[0]
0%| | 0/1591 [00:00<?, ?it/s]
삭제할 자료¶
CASE 2_10, 2_11, 34_01, 40_01, 40_02, 44_01, 52_01, 56_01, 60_20~34, 63_01, 64_01 : 환경자료 결측
CASE 8, 9, 22, 23, 26, 30, 31, 49, 59, 71, 72, 73 : 환경자료 결측
CASE 35_01, 41_01, 44_02, 45_01, 52_02, 53_01, 56_02, 57_01, 63_02 : 부분결측(제거)
CASE 34, 35, 48 : EC 결측
CASE 32_15, 51_11 : Co2 이상
In [7]:
env_na_p=[f"CASE{i}" for i in set([f"60_{i}" for i in range(20,34+1)]).union( set(['02_10','02_11','34_01','40_01','40_02','44_01','52_01','56_01','63_01','64_01']))] env_na_a=[f"CASE{i}" for i in ['08','09','22','23','26','30','31','49','59','71','72','73']] partial_na=[f"CASE{i}" for i in ["35_01","41_01","44_02","45_01","52_02","53_01","57_01","63_02"]] ec_na=[f"CASE{i}" for i in ["34","35","48"]] tr_df['na_label']=False for i in (env_na_p+env_na_a+partial_na+ec_na): tr_df.loc[tr_df['img_dirs'].str.contains(i),"na_label"]=True
In [8]:
for i, filename in tqdm(enumerate(tr_df.img_dirs)): temp_df=pd.read_csv(filename.replace('image','meta').replace('jpg','png').replace('png','csv')) time_df=time_df.sort_values('시간') temp_df.시간=pd.to_datetime(temp_df.시간) temp_df['청색광추정광량']=(temp_df['총추정광량']- temp_df['백색광추정광량']+temp_df['적색광추정광량'])[temp_df['청색광추정광량'].isna()] aftn_co2=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(9,19))),'CO2관측치'].quantile(.5) night_co2=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(19,23))+list(range(0,5))),'CO2관측치'].quantile(.5) co2_ratio=aftn_co2/night_co2 # 1보다 낮으면 생육단계, 1보다 크면 발아단계 zero_ec_cnt=sum(temp_df['EC관측치']==0) disease_signal=co2_ratio*zero_ec_cnt aftn_ec=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(10,15))),'EC관측치'].quantile(.5) night_ec1=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(22,23))),'EC관측치'].quantile(.5) night_ec2=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(3,5))),'EC관측치'].quantile(.5) m_temp=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(10,18))),'내부온도관측치'].mean(skipna=True) m_humidity=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(10,15))),'내부습도관측치'].mean(skipna=True) # 결측자료 처리 if np.isnan(aftn_ec): if tr_df['del_leaf'][i]>10: aftn_ec=night_ec1*1.5 else: aftn_ec=night_ec2 if np.isnan(m_temp): m_temp=temp_df['내부온도관측치'].mean(skipna=True) if np.isnan(m_humidity): m_humidity=temp_df['내부습도관측치'].mean(skipna=True) if np.isnan(night_ec1): night_ec1=0 if np.isnan(co2_ratio): if tr_df.pred_leaf_mean[i]>50: co2_ratio = 0.5 else: co2_ratio = 1.5 disease_signal=co2_ratio*zero_ec_cnt tr_df.loc[i,['co2_ratio','zero_ec_cnt','disease_signal', 'aftn_ec','night_ec1','night_ec2','m_temp','m_humidity']]=\ co2_ratio, zero_ec_cnt, disease_signal, aftn_ec, night_ec1, night_ec2, m_temp, m_humidity
0it [00:00, ?it/s]
In [13]:
for i, filename in tqdm(enumerate(te_df.img_dirs)): temp_df=pd.read_csv(filename.replace('image','meta').replace('jpg','png').replace('png','csv')) time_df=time_df.sort_values('시간') temp_df.시간=pd.to_datetime(temp_df.시간) temp_df['청색광추정광량']=(temp_df['총추정광량']- temp_df['백색광추정광량']+temp_df['적색광추정광량'])[temp_df['청색광추정광량'].isna()] aftn_co2=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(9,19))),'CO2관측치'].quantile(.5) night_co2=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(19,23))+list(range(0,5))),'CO2관측치'].quantile(.5) co2_ratio=aftn_co2/night_co2 # 1보다 낮으면 생육단계, 1보다 크면 발아단계 zero_ec_cnt=sum(temp_df['EC관측치']==0) aftn_ec=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(10,15))),'EC관측치'].quantile(.5) night_ec1=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(22,23))),'EC관측치'].quantile(.5) night_ec2=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(3,5))),'EC관측치'].quantile(.5) m_temp=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(10,18))),'내부온도관측치'].mean(skipna=True) m_humidity=temp_df.loc[temp_df.시간.dt.hour.isin(list(range(10,15))),'내부습도관측치'].mean(skipna=True) # 결측자료 처리 if np.isnan(aftn_ec): if te_df['del_leaf'][i]>10: aftn_ec=night_ec1*1.5 else: aftn_ec=night_ec2 if np.isnan(m_temp): m_temp=temp_df['내부온도관측치'].mean(skipna=True) if np.isnan(m_humidity): m_humidity=temp_df['내부습도관측치'].mean(skipna=True) if np.isnan(night_ec1): night_ec1=0 if np.isnan(co2_ratio): if te_df.pred_leaf_mean[i]>50: co2_ratio = 0.5 else: co2_ratio = 1.5 disease_signal=co2_ratio*zero_ec_cnt te_df.loc[i,['co2_ratio','zero_ec_cnt','disease_signal', 'aftn_ec','night_ec1','night_ec2','m_temp','m_humidity']]=\ co2_ratio, zero_ec_cnt, disease_signal, aftn_ec, night_ec1, night_ec2, m_temp, m_humidity
0it [00:00, ?it/s]
청경채 무게 추정¶
현재시점 예측¶
In [14]:
ranger=r(''' library(ranger) f=function(df,formula){ rf=ranger(as.formula(formula),data=df,seed=42,importance = "permutation") return(rf) } ''') predict=r(''' library(ranger) f=function(rf,df){ pred=predict(rf,df)$pred return(pred) } ''') importance=r(''' library(ranger) f=function(rf){ return(rf$variable.importance) } ''')
미래시점 예측¶
In [24]:
pred_rf=ranger(tr_df[~tr_df.na_label].drop(['date','now_weight'],axis=1).dropna(), formula="""leaf_weight~pred_leaf_mean+ del_leaf+co2_ratio+zero_ec_cnt+disease_signal+ aftn_ec+night_ec1+night_ec2+m_temp+m_humidity""")
In [25]:
plt.scatter(tr_df[~tr_df.na_label].drop(['date','now_weight'],axis=1).dropna().leaf_weight, predict(pred_rf,tr_df[~tr_df.na_label].drop(['date','now_weight'],axis=1).dropna()))
Out[25]:
<matplotlib.collections.PathCollection at 0x7fd258a421d0>
In [26]:
plt.barh(['pred_leaf_mean', 'del_leaf', 'co2_ratio','zero_ec_cnt','disease_signal','aftn_ec', 'night_ec1','night_ec2','m_temp','m_humidity'],importance(pred_rf))
Out[26]:
<BarContainer object of 10 artists>
In [27]:
submit=pd.read_csv(f"{main_dir}/sample_submission.csv") submit['leaf_weight']=predict(pred_rf,te_df.dropna()) submit.to_csv(f"/root/jupyter/데이콘/청경채/submit_4.csv",index=False)
In [ ]: