Browse Source

read hdfs

丁云鹏 5 tháng trước cách đây
mục cha
commit
a81d5c0f16

+ 5 - 2
recommend-model-produce/src/main/python/Dockerfile

@@ -1,8 +1,11 @@
 FROM registry.baidubce.com/paddlepaddle/paddle:2.6.1
 RUN pip freeze > requirements.txt
 ENV PYTHONUNBUFFERED=1
+ENV GLOG_v=4
 ENV JAVA_HOME=/app/env/java8
-ENV PATH=$PATH:$JAVA_HOME/bin
+ENV HADOOP_HOME=/app/env/hadoop-3.2.4
+ENV PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
 WORKDIR /app
 COPY . /app
-RUN pip3 install -i https://mirrors.aliyun.com/pypi/simple/ --no-cache-dir -r requirements.txt
+RUN pip3 install -i https://mirrors.aliyun.com/pypi/simple/ --no-cache-dir -r requirements.txt
+CMD python tools/static_ps_trainer.py -m models/dssm/config_ps.yaml

+ 2 - 2
recommend-model-produce/src/main/python/tools/static_ps_trainer_v2.py

@@ -261,7 +261,7 @@ class Main(object):
 
         print_step = int(config.get("runner.print_interval"))
 
-        debug = config.get("runner.dataset_debug", True)
+        debug = config.get("runner.dataset_debug", False)
         if config.get("runner.need_dump"):
             debug = True
             dump_fields_path = "{}/{}".format(
@@ -279,7 +279,7 @@ class Main(object):
             fetch_info=fetch_info,
             print_period=print_step,
             debug=debug)
-        
+
 
     def heter_train_loop(self, epoch):
         logger.info(