p4/Dockerfile.datanode

+FROM p4-hdfs
+CMD export CLASSPATH=`$HADOOP_HOME/bin/hdfs classpath --glob` && \
+    hdfs datanode -D dfs.datanode.data.dir=/var/datanode -fs hdfs://boss:9000
\ No newline at end of file

p4/Dockerfile.hdfs

+FROM ubuntu:24.04
+RUN apt-get update; apt-get install -y wget curl openjdk-11-jdk python3-pip iproute2 nano
+
+# HDFS
+RUN wget https://dlcdn.apache.org/hadoop/common/hadoop-3.3.6/hadoop-3.3.6.tar.gz; tar -xf hadoop-3.3.6.tar.gz; rm hadoop-3.3.6.tar.gz
+
+ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk-amd64
+ENV PATH="${PATH}:/hadoop-3.3.6/bin"
+ENV HADOOP_HOME=/hadoop-3.3.6

p4/Dockerfile.mysql

+FROM mysql:8.4.0-oraclelinux8
+
+WORKDIR /
+#COPY init.sql /docker-entrypoint-initdb.d/
+COPY hdma-wi-2021.sql.gz /docker-entrypoint-initdb.d/
+RUN gzip -d /docker-entrypoint-initdb.d/hdma-wi-2021.sql.gz
+CMD ["mysqld"]
+

p4/Dockerfile.namenode

+FROM p4-hdfs
+CMD export CLASSPATH=`$HADOOP_HOME/bin/hdfs classpath --glob` && \
+    hdfs namenode -format &&\
+    hdfs namenode -D dfs.namenode.stale.datanode.interval=10000 -D dfs.namenode.heartbeat.recheck-interval=30000 -fs hdfs://boss:9000

p4/client.py

+import sys
+import grpc
+import argparse
+from concurrent import futures
+import lender_pb2, lender_pb2_grpc
+import pandas as pd
+
+
+parser = argparse.ArgumentParser(description="argument parser for p4 clinet")
+
+
+parser.add_argument("mode", help="which action to take", choices=["DbToHdfs","BlockLocations","CalcAvgLoan"])
+
+parser.add_argument("-c", "--code", type=int, default=0, help="county code to query average loan amount in CalcAvgLoan mode")
+parser.add_argument("-f", "--file", type=str, default="", help="file path for BlockLocation")
+args = parser.parse_args()
+
+channel = grpc.insecure_channel("server:5000")
+stub = lender_pb2_grpc.LenderStub(channel)
+if args.mode == "DbToHdfs":        
+    resp = stub.DbToHdfs(lender_pb2.Empty())
+    print(resp.status)
+elif args.mode == "CalcAvgLoan":
+    resp = stub.CalcAvgLoan(lender_pb2.CalcAvgLoanReq(county_code=args.code))
+    if resp.error:
+        print(f"error: {resp.error}")
+    else:
+        print(resp.avg_loan)
+        print(resp.source)
+elif args.mode == "BlockLocations":
+    resp = stub.BlockLocations(lender_pb2.BlockLocationsReq(path=args.file)) 
+    if resp.error:
+        print(f"error: {resp.error}")
+    else:
+        print(resp.block_entries)
+
+
+

p5/.gitignore

+venv/*
+nb/.ipynb_checkpoints/*
+nb/data/*
+nb/metastore_db/*
+nb/derby.log
\ No newline at end of file

p5/datanode.Dockerfile

+FROM p5-base
+CMD ["bash", "-c", "hdfs datanode -fs hdfs://nn:9000"]