[python練習] numpy

引用

import numpy as np;

1維陣列

a = np.array([1, 2, 3]);

2維陣列

a = np.array([ [1, 2, 3], [4, 5, 6] ]);

3維陣列

a = np.array([ [ [1, 2, 3], [4, 5, 6] ], [ [7, 8, 9], [10, 11, 12] ] ]);

內積 (vector dot)

vector dot

v1 = np.array([1, 2, 3]);
v2 = np.array([4, 5, 6]);
v3 = np.dot(v1,v2); #v3 = 1*4 + 2*5 + 3*6 = 4 + 10 + 18 = 32
print(v3);

矩陣相乘 (matrix multiplication)

matrix_multiplication

m1 = np.array([ [1, 2, 3], [4, 5, 6] ]);      #size: 2*3
m2 = np.array([ [1, 2 ], [3, 4], [5, 6] ]);   #size: 3*2
m3 = np.matmul(m1,m2); # m3= [ [ 1*1+2*3+3*5 1*2+2*4+3*6] [ 4*1+5*3+6*5 4*2+5*4+6*6 ] ] = [ [ 22 28 ] [ 49 64 ] ]
print(m3);

元素相乘 (multiply)

  • 不常用
m1 = np.arange(1,5).reshape(2,2); # m1 = [ [ 1 2 ] [ 3 4 ] ]
m2 = np.arange(0,4).reshape(2,2); # m2 = [ [ 0 1 ] [ 2 3 ] ]
m3 = np.multiply(m1,m2);          # m3 = [ [ 1*0 2*1 ] [ 3*2 4*3 ] ] = [ [ 0 2 ] [ 6 12 ] ]
print(m3);

原始碼

[python練習] pyplot

import matplotlib.pyplot as plt
import numpy as np

#產生隨機資料
x_data = np.linspace(-100, 100, 300)[:, np.newaxis];
#加入雜訊
noise = np.random.normal(0, 300, x_data.shape);
y_data = np.square(x_data) - 0.5 + noise;

plt.plot(x_data, y_data, 'g.');

#動態繪製sin曲線
x = np.arange(-100,100) 
for i in range(1000):
  #嘗試移除畫過的sin曲線,第一次會發生錯誤,因此用try/except
  try:
    lines.pop(0).remove()
  except Exception:
    pass
  #重新計算y值
  y = 1000*np.sin((x+i) * np.pi/ 9.0)
  #繪製sin曲線
  lines=plt.plot(x,y,'r-')
  #延遲0.1秒
  plt.pause(0.1)

原始碼

使用EDNS跟DNSSec保護DNS資料

DNS伺服器扮演著將IP與網域對應的工作.
但因為當初DNS設計並沒有考量安全性的設計,導致至今有許多DNS偽造的情況.
為瞭解決上述問題,衍生出了DNSSec以及EDNS等相關技術的發展.

然而DNS已經行之有年,許多DNS Server並沒有落實EDNS.
所以過去有一段時間,公用的DNS Server對於未支援EDNS的伺服器會採用二次查詢的方式.
也就是第一次會使用EDNS查詢,失敗後改用傳統DNS方式查詢.
如此一來就可以相容兩種系統,但是也造成EDNS遲遲無法普及.

因此幾個公用的DNS Server(如Google 8.8.8.8, GloudFlare 1.1.1.1)約定在2019/2/1這天要來測試只做EDNS查詢,並將這一天稱為DNS Flag Day.
所以當DNS Server不支援EDNS時,將會造成網域名稱無法順利解析.

為避免上述狀況,本研究採用下列方式,開啟EDNS及DNSSec之設定.

下面是有關於如何將DNS提升至EDNS的方法:

1.確定網域名稱,並準備好網域設定檔

DOMAIN=cbe.tw
# 網域設定檔: ${DOMAIN}.txt

2.建立公私鑰
2.1.KSK

KSK_KEY=$(dnssec-keygen -r /dev/urandom -f KSK -a RSASHA512 -b 2048 -n ZONE ${DOMAIN})

2.2.ZSK

ZSK_KEY=$(dnssec-keygen -r /dev/urandom -a RSASHA512 -b 1024 -n ZONE ${DOMAIN})

3.將key附加到網域設定檔 ( 也就是${DOMAIN}.txt )

cat K${DOMAIN}*.key >> ${DOMAIN}.txt

4.產生加簽後的網域設定檔 ( )

dnssec-signzone -o ${DOMAIN} -k ${KSK_KEY}.key ${DOMAIN}.txt ${ZSK_KEY}.key

上述步驟將會產生:
* 加簽過後的網域設定檔,檔名會是 ${DOMAIN}.txt.signed
* DS紀錄檔(DS records),檔名會是 dsset-${DOMAIN}.

5.啟用DNSSec
* 將${DOMAIN}.txt.signed放到 DNS Server上 (如Bind9 or CoreDNS).
* 到網域註冊商上加入DS records.

* 開啟TCP 53 port,並做好對應的NAT.
6.檢測
* https://dnsflagday.net/#domain-holders

* https://dnssec-analyzer.verisignlabs.com/

* http://dnsviz.net/

移除XQuartz

launchctl unload /Library/LaunchAgents/org.macosforge.xquartz.startx.plist && \
sudo launchctl unload /Library/LaunchDaemons/org.macosforge.xquartz.privileged_startx.plist && \
sudo rm -rf /opt/X11* /Library/Launch*/org.macosforge.xquartz.* /Applications/Utilities/XQuartz.app /etc/*paths.d/*XQuartz  && \
sudo pkgutil --forget org.macosforge.xquartz.pkg  && \
rm -rf ~/.serverauth* && rm -rf ~/.Xauthorit* && rm -rf ~/.cache && rm -rf ~/.rnd && \
rm -rf ~/Library/Caches/org.macosforge.xquartz.X11 && rm -rf ~/Library/Logs/X11

refer: https://gist.github.com/pwnsdx/d127873e24cef159d4d603accaf37ee4

經測試,可移除使用dmg安裝的XQuartz

修改docker0網段

其中10.172.254.254是Getway,不要自作主張改成 192.168.1.0/24 or 10.172.0.0之類的..

yum install bridge-utils -y

service docker stop
ip link set dev docker0 down
brctl delbr docker0
iptables -t nat -F POSTROUTING


brctl addbr docker0
ip addr add 10.172.254.254/16 dev docker0
ip link set dev docker0 up


cat << EOF > /etc/docker/daemon.json
{
  "bip": "10.172.254.254/16"
}
EOF

systemctl daemon-reload
systemctl restart docker.service
reboot #如果有k8s則需要重開機

refer: https://blog.yowko.com/docker-172-17-ip/

tensorflow 初測

#TAG=latest-py3
TAG=latest-gpu-py3

#準備資料
rm -rf $PWD/test
mkdir -p $PWD/test
cd $PWD/test
git clone https://github.com/purocean/tensorflow-simple-captcha.git
cd tensorflow-simple-captcha

#前處理
docker run \
  --runtime=nvidia \
  -it --rm \
  -v $PWD:/data \
  -w /data \
  tensorflow/tensorflow:$TAG python3 prepare.py

#訓練
time docker run \
  --runtime=nvidia \
  -it --rm \
  -v $PWD:/data \
  -w /data \
  tensorflow/tensorflow:$TAG python3 train.py

#測試
docker run \
  --runtime=nvidia \
  -it --rm \
  -v $PWD:/data \
  -w /data \
  tensorflow/tensorflow:$TAG bash -c 'python3 work.py < test.jpg'

在centos 7上安裝nvidia docker

#安裝 docker-ce
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
yum install docker-ce-18.06.0.ce -y
systemctl enable docker && systemctl start docker


# 移除舊版nvidia-docker
docker volume ls -q -f driver=nvidia-docker | xargs -r -I{} -n1 docker ps -q -a -f volume={} | xargs -r docker rm -f
sudo yum remove nvidia-docker

# 加入repositories
distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
curl -s -L https://nvidia.github.io/nvidia-docker/$distribution/nvidia-docker.repo | sudo tee /etc/yum.repos.d/nvidia-docker.repo

# 安裝 nvidia-docker2
sudo yum install -y nvidia-docker2
sudo pkill -SIGHUP dockerd

# 測試
docker run --runtime=nvidia --rm nvidia/cuda:9.0-base nvidia-smi

refer: https://github.com/NVIDIA/nvidia-docker

k8s安裝ingress nginx

ingress-nginx最新版已經不包含default-http-backend.
因此安裝0.20.0

#安裝ingress-nginx
wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/nginx-0.20.0/deploy/mandatory.yaml
sed -i 's/serviceAccountName: nginx-ingress-serviceaccount/hostNetwork: true\n      serviceAccountName: nginx-ingress-serviceaccount/g' mandatory.yaml
kubectl apply -f mandatory.yaml
rm -f mandatory.yaml*
kubectl get pod --all-namespaces

#修改ingress-nginx
NODE_COUNT=$(kubectl get nodes | grep -v master | grep -v STATUS | wc -l)
echo $NODE_COUNT
if [ $NODE_COUNT -gt 1 ] ; then
kubectl -n ingress-nginx patch deployment default-http-backend --patch $(echo "{\"spec\":{\"replicas\":$NODE_COUNT}}")
kubectl -n ingress-nginx patch deployment nginx-ingress-controller --patch $(echo "{\"spec\":{\"replicas\":$NODE_COUNT}}")
fi
kubectl get pods -n ingress-nginx -o wide

#更換自製 http-backend image
DOMAIN=ssl.cbe.tw
kubectl -n ingress-nginx patch deployment default-http-backend --patch "{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"default-http-backend\",\"resources\":{\"limits\":{\"cpu\":\"100m\",\"memory\":\"200Mi\"},\"requests\":{\"cpu\":\"100m\",\"memory\":\"200Mi\"}},\"image\":\"slanla/apache-defaultbackend\",\"ports\":[{\"containerPort\":8080,\"protocol\":\"TCP\"}],\"env\":[{\"name\":\"LETSENCRYPT_PROXYPASS_URL\",\"value\":\"http://$DOMAIN/.well-known/acme-challenge/ connectiontimeout=15 timeout=30\"},{\"name\":\"LETSENCRYPT_PROXYPASSREVERSE_URL\",\"value\":\"http://$DOMAIN/.well-known/acme-challenge/\"}],\"livenessProbe\":{\"httpGet\":{\"path\":\"/healthz\",\"port\":8080,\"scheme\":\"HTTP\"},\"initialDelaySeconds\":30,\"timeoutSeconds\":5,\"periodSeconds\":10,\"successThreshold\":1,\"failureThreshold\":3}}]}}}}"
kubectl get pods -n ingress-nginx -o wide