[root@localhost cloud_images]# guestfish -rw -a vhuser-test1.qcow2 guestfish: cannot mix --ro and --rw options [root@localhost cloud_images]# guestfish -rw -a vhuser-test1.qcow2 guestfish: cannot mix --ro and --rw options [root@localhost cloud_images]# guestfish --rw -a vhuser-test1.qcow2 Welcome to guestfish, the guest filesystem shell for editing virtual machine filesystems and disk images. Type: ‘help’ for help on commands ‘man’ to read the manual ‘quit’ to quit the shell ><fs>run ><fs> list-filesystems /dev/sda1: vfat /dev/sda2: xfs ><fs> ls /dev/sda2 libguestfs: error: ls0: ls0_stub: you must call 'mount' first to mount the root filesystem ><fs> mount /dev/sda2 / ><fs> ls error: incorrect number of arguments usage: ls directory type 'help ls' for more help on ls ><fs> ls / bin boot dev etc home lib lib64 media mnt opt proc root run sbin srv sys tmp usr var ><fs>
https://blahcat.github.io/2018/01/07/building-a-debian-stretch-qemu-image-for-aarch64/
ls -l /dev/disk/
dracut:/# ls /dev/disk/ by-partlabel by-partuuid by-path by-uuid dracut:/# ls /dev/disk/by-path/ platform-4010000000.pcie-pci-0000:00:03.0 platform-4010000000.pcie-pci-0000:00:03.0-part1 platform-4010000000.pcie-pci-0000:00:03.0-part2 dracut:/# ls /dev/disk/by-partlabel/ EFI\x20System\x20Partition dracut:/# ls /dev/disk/by-partuuid/ 07371592-36ba-4a78-95a0-544036ae95c8 5056e898-b14f-47b4-9539-45bc54dbd7d6 dracut:/# ls /dev/disk/by-partuuid/ -al total 0 drwxr-xr-x 2 root 0 80 Nov 4 12:53 . drwxr-xr-x 6 root 0 120 Nov 4 12:53 .. lrwxrwxrwx 1 root 0 10 Nov 4 12:53 07371592-36ba-4a78-95a0-544036ae95c8 -> ../../vda2 lrwxrwxrwx 1 root 0 10 Nov 4 12:53 5056e898-b14f-47b4-9539-45bc54dbd7d6 -> ../../vda1 dracut:/# ls ../../vda1 ls: cannot access ../../vda1: No such file or directory dracut:/#
qemu版本
[root@localhost cloud_images]# qemu-system-aarch64 -version
QEMU emulator version 5.1.90 (v5.2.0-rc0)
Copyright (c) 2003-2020 Fabrice Bellard and the QEMU Project developers
[root@localhost cloud_images]#
無法識別啟動盤
將啟動參數改為uuid
虛擬機
[root@localhost ~]# cat /proc/cmdline
console=ttyAMA0 root=UUID=6a09973e-e8fd-4a6d-a8c0-1deb9556f477
[root@localhost ~]# ls
[root@localhost ~]# cat /etc/fstab
#
# /etc/fstab
# Created by anaconda on Wed Apr 22 10:08:46 2020
#
# Accessible filesystems, by reference, are maintained under '/dev/disk'
# See man pages fstab(5), findfs(8), mount(8) and/or blkid(8) for more info
#
UUID=6a09973e-e8fd-4a6d-a8c0-1deb9556f477 / xfs defaults 0 0
UUID=7ADD-F946 /boot/efi vfat defaults,uid=0,gid=0,umask=0077,shortname=winnt 0 0
[root@localhost ~]# ls /dev/disk/by-uuid/
6a09973e-e8fd-4a6d-a8c0-1deb9556f477 7ADD-F946
[root@localhost ~]#
准備：需要QEMU_EFI.fd 文件，可以安裝edk2.git-aarch64獲取：
wget https://www.kraxel.org/repos/firmware.repo -O /etc/yum.repos.d/firmware.repo
yum -y install edk2.git-aarch64
或者離線安裝，在有外網的環境下訪問https://www.kraxel.org/repos/jenkins/edk2/，獲取rpm包並拷貝至目標服務器系統相應位置。執行如下命令離線安裝edk2
rpm -ivh edk2.git-aarch64*.rpm
安裝完畢后：/usr/share/edk2.git/aarch64/QEMU_EFI.fd 拷貝到實際想要使用的路徑下
QEMU_EFI.fd 下載
https://releases.linaro.org/components/kernel/uefi-linaro/16.02/release/qemu64/
[root@localhost cloud_images]# bash kp.sh 3h 3h 3h 3h 3h 3h Failed to set MokListRT: Invalid Parameter Something has gone seriously wrong: import_mok_state() failed : Invalid Parameter [root@localhost cloud_images]#
#!/bin/bash qemu-system-aarch64 \ -smp 2 \ -m 1024 \ -M virt \ -cpu cortex-a57 \ -bios /usr/share/qemu-efi-aarch64/QEMU_EFI.fd \ -nographic \ -device virtio-blk-device,drive=image \ -drive if=none,id=image,file=ubuntu-16.04-server-cloudimg-arm64-uefi1.img,format=qcow2 \ -device virtio-blk-device,drive=cloud \ -drive if=none,id=cloud,file=cloud.img,format=qcow2 \ -device e1000,netdev=net0 \ -netdev user,id=net0,hostfwd=tcp:127.0.0.1:2222-:22
在使用qemu時，客戶機（虛擬機）與主機之間通訊可通過hostfwd參數指定轉發端口來實現，
-netdev user,id=n0,hostfwd=[tcp|udp]:[主機ip]:主機端口-[客戶機ip]:客戶機端口
這種方法可以在主機的qemu進程監聽一個端口，主機可通過這個端口與客戶機對應的端口通訊。
這時問題就來了，如果我要在客戶機開啟一系列監聽端口，就必須要寫一大串hostfwd參數才能實現功能，即使是一段連續的端口號也一樣。
比如要在客戶機對外開放ftp服務，ftp主動模式的命令端口是21，數據端口是20，就必須寫兩個hostfwd參數並用逗號隔開：
-netdev user,id=n0,hostfwd=::1020-:20,hostfwd=::1021-:21
[root@localhost cloud_images]# netstat -ntplu | grep 6002 tcp 0 0 127.0.0.1:6002 0.0.0.0:* LISTEN 38528/qemu-system-a [root@localhost cloud_images]#
虛擬機1 ---不需要設置-bios QEMU_EFI.fd
kernel和initrd參數都是從qcow2拷貝,沒有daemonize參數
-chardev socket,path=/tmp/vm2_qga0.sock,server,nowait,id=vm2_qga0 -device virtio-serial \ -device virtserialport,chardev=vm2_qga0,name=org.qemu.guest_agent.2 -daemonize \ -monitor unix:/tmp/vm2_monitor.sock,server,nowait -net nic,macaddr=00:00:00:08:e8:aa,addr=1f \
qemu-system-aarch64 -name vm2 -nographic \ -enable-kvm -M virt -cpu host -smp 2 -m 4096 \ -global virtio-blk-device.scsi=off \ -device virtio-scsi-device,id=scsi \ -kernel vmlinuz-4.18 --append "console=ttyAMA0 root=UUID=6a09973e-e8fd-4a6d-a8c0-1deb9556f477" \ -initrd initramfs-4.18 \ -object memory-backend-file,id=mem,size=4096M,mem-path=/mnt/huge,share=on \ -numa node,memdev=mem -mem-prealloc -drive file=vhuser-test1.qcow2 \ -chardev socket,path=/tmp/vm2_qga0.sock,server,nowait,id=vm2_qga0 -device virtio-serial \ -device virtserialport,chardev=vm2_qga0,name=org.qemu.guest_agent.2 \ -monitor unix:/tmp/vm2_monitor.sock,server,nowait -net nic,macaddr=00:00:00:08:e8:aa,addr=1f \ -net user,hostfwd=tcp:127.0.0.1:6002-:22 \ -vnc :10 #./build/vhost-switch -l 0-3 -n 4 --huge-dir /dev/hugepages --socket-mem 1024 --log-level 8 -w 0000:07:00.1 -- --socket-file /tmp/vhost-user1 --client -p 0x1 --stats 20 ''' -chardev socket,id=char0,path=/tmp/vhost-user1,server \ -netdev type=vhost-user,id=netdev0,chardev=char0,vhostforce \ -device virtio-net-pci,netdev=netdev0,mac=52:54:00:00:00:01,mrg_rxbuf=on,rx_queue_size=1024,tx_queue_size=1024 \ '''
[FAILED] Failed to start OpenSSH server daemon. See 'systemctl status sshd.service' for details. [ OK ] Started System Logging Service. [ OK ] Started Dynamic System Tuning Daemon. [ OK ] Started Postfix Mail Transport Agent. [ OK ] Stopped OpenSSH server daemon. Starting OpenSSH server daemon... [FAILED] Failed to start OpenSSH server daemon. See 'systemctl status sshd.service' for details. [ OK ] Stopped OpenSSH server daemon. Starting OpenSSH server daemon... [FAILED] Failed to start OpenSSH server daemon. See 'systemctl status sshd.service' for details. [FAILED] Failed to start GSSAPI Proxy Daemon. See 'systemctl status gssproxy.service' for details. [ OK ] Reached target NFS client services. [ OK ] Reached target Remote File Systems (Pre). [ OK ] Reached target Remote File Systems. Starting Permit User Sessions... Starting Crash recovery kernel arming... [ OK ] Started Permit User Sessions. [ OK ] Started Getty on tty1. [ OK ] Started Command Scheduler. [ OK ] Started Serial Getty on ttyAMA0. [ OK ] Reached target Login Prompts. [ OK ] Reached target Multi-User System. Starting Update UTMP about System Runlevel Changes... [ OK ] Started Update UTMP about System Runlevel Changes. [FAILED] Failed to start Crash recovery kernel arming. See 'systemctl status kdump.service' for details
[root@localhost ~]# telnet 127.0.0.1 6002 Trying 127.0.0.1... Connected to 127.0.0.1. Escape character is '^]'. Connection closed by foreign host.
更改啟動參數
qemu-system-aarch64 -name vm2 -nographic \ -enable-kvm -M virt -cpu host -smp 2 -m 4096 \ -global virtio-blk-device.scsi=off \ -device virtio-scsi-device,id=scsi \ -kernel vmlinuz-4.18 --append "console=ttyAMA0 root=UUID=6a09973e-e8fd-4a6d-a8c0-1deb9556f477" \ -initrd initramfs-4.18 \ -object memory-backend-file,id=mem,size=4096M,mem-path=/mnt/huge,share=on \ -numa node,memdev=mem -mem-prealloc -drive file=vhuser-test1.qcow2 \ -chardev socket,path=/tmp/vm2_qga0.sock,server,nowait,id=vm2_qga0 -device virtio-serial \ -device virtserialport,chardev=vm2_qga0,name=org.qemu.guest_agent.2 \ -monitor unix:/tmp/vm2_monitor.sock,server,nowait -net nic,macaddr=00:00:00:08:e8:aa,addr=1f \ -device e1000,netdev=net0 \ -netdev user,id=net0,hostfwd=tcp:127.0.0.1:6002-:22 \ -vnc :10
netsta -pan
warning, got bogus unix line. warning, got bogus unix line. warning, got bogus unix line. warning, got bogus unix line. warning, got bogus unix line. warning, got bogus unix line. warning, got bogus unix line. warning, got bogus unix line. warning, got bogus unix line. warning, got bogus unix line. [root@localhost ~]# lsof -bash: lsof: command not found [root@localhost ~]# systemctl status sshd ● sshd.service - OpenSSH server daemon Loaded: loaded (/usr/lib/systemd/system/sshd.service; enabled; vendor preset: enabled) Active: activating (auto-restart) (Result: exit-code) since Thu 2020-11-05 02:13:10 UTC; 41s ago Docs: man:sshd(8) man:sshd_config(5) Process: 1205 ExecStart=/usr/sbin/sshd -D $OPTIONS (code=exited, status=1/FAILURE) Main PID: 1205 (code=exited, status=1/FAILURE) Nov 05 02:13:10 localhost.localdomain systemd[1]: sshd.service: main process ... Nov 05 02:13:10 localhost.localdomain systemd[1]: Failed to start OpenSSH ser... Nov 05 02:13:10 localhost.localdomain systemd[1]: Unit sshd.service entered f... Nov 05 02:13:10 localhost.localdomain systemd[1]: sshd.service failed. Hint: Some lines were ellipsized, use -l to show in full. [root@localhost ~]# systemctl restart sshd Job for sshd.service failed because the control process exited with error code. See "systemctl status sshd.service" and "journalctl -xe" for details. [root@localhost ~]# journalctl -xe -- Defined-By: systemd -- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel -- -- Unit sshd.service has finished shutting down. Nov 05 02:14:07 localhost.localdomain systemd[1]: Starting OpenSSH server daemon -- Subject: Unit sshd.service has begun start-up -- Defined-By: systemd -- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel -- -- Unit sshd.service has begun starting up. Nov 05 02:14:07 localhost.localdomain sshd[1217]: /etc/ssh/sshd_config: Permissi Nov 05 02:14:07 localhost.localdomain systemd[1]: sshd.service: main process exi Nov 05 02:14:07 localhost.localdomain systemd[1]: Failed to start OpenSSH server -- Subject: Unit sshd.service has failed -- Defined-By: systemd -- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel -- -- Unit sshd.service has failed. -- -- The result is failed. Nov 05 02:14:07 localhost.localdomain systemd[1]: Unit sshd.service entered fail Nov 05 02:14:07 localhost.localdomain systemd[1]: sshd.service failed. Nov 05 02:14:07 localhost.localdomain polkitd[712]: Unregistered Authentication [root@localhost ~]# ip a 1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000 link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00 inet 127.0.0.1/8 scope host lo valid_lft forever preferred_lft forever inet6 ::1/128 scope host valid_lft forever preferred_lft forever 2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000 link/ether 00:00:00:08:e8:aa brd ff:ff:ff:ff:ff:ff inet6 fe80::200:ff:fe08:e8aa/64 scope link valid_lft forever preferred_lft forever
[FAILED] Failed to start GSSAPI Proxy Daemon. See 'systemctl status gssproxy.service' for details. [ OK ] Reached target NFS client services. [ OK ] Reached target Remote File Systems (Pre). [ OK ] Reached target Remote File Systems. Starting Permit User Sessions... [ OK ] Started Permit User Sessions. [ OK ] Started Getty on tty1. [ OK ] Started Command Scheduler. [ OK ] Started Serial Getty on ttyAMA0. [ OK ] Reached target Login Prompts.
參考https://cloud.tencent.com/developer/article/1634850
由於特殊場景需求，在制作pxe ramdisk時使用最小化安裝，沒有安裝rng-tools服務，現安裝rng-tools補充熵池，並設置開機自動運行該服務： [root@localhost ~]# yum install rng-tools [root@localhost ~]# systemctl enable rng-tools
[ OK ] Started Preprocess NFS configuration. [ OK ] Started Import network configuration from initramfs. Starting Create Volatile Files and Directories... [ OK ] Started Create Volatile Files and Directories. Starting Security Auditing Service... [ OK ] Started Security Auditing Service. Starting Update UTMP about System Boot/Shutdown... [ OK ] Started Update UTMP about System Boot/Shutdown. [ OK ] Reached target System Initialization. [ OK ] Started Daily Cleanup of Temporary Directories. [ OK ] Reached target Timers. [ OK ] Listening on RPCbind Server Activation Socket. Starting RPC bind service... [ OK ] Listening on D-Bus System Message Bus Socket. [ OK ] Reached target Sockets. [ OK ] Reached target Basic System. [ OK ] Started irqbalance daemon. Starting OpenSSH Server Key Generation... Starting LSB: Bring up/down networking... [ OK ] Started D-Bus System Message Bus. Starting NTP client/server... Starting GSSAPI Proxy Daemon... Starting Authorization Manager... Starting Login Service... [ OK ] Started Hardware RNG Entropy Gatherer Daemon. Starting Dump dmesg to /var/log/dmesg... [ OK ] Started RPC bind service. [ OK ] Started Dump dmesg to /var/log/dmesg. [ OK ] Started NTP client/server. [ OK ] Started Login Service. [ OK ] Started Authorization Manager. [ OK ] Started OpenSSH Server Key Generation. [ 4.592472] random: crng init done [ 4.604657] random: 7 urandom warning(s) missed due to ratelimiting [ OK ] Started GSSAPI Proxy Daemon. [ OK ] Reached target NFS client services. [ OK ] Reached target Remote File Systems (Pre). [ OK ] Reached target Remote File Systems. Starting Permit User Sessions... [ OK ] Started Permit User Sessions. [ OK ] Started Serial Getty on ttyAMA0. [ OK ] Started Getty on tty1. [ OK ] Reached target Login Prompts. [ OK ] Started Command Scheduler. CentOS Linux 7 (AltArch)
install rng-tools 后netstat也正常了
[root@localhost ~]# systemctl restart sshd Job for sshd.service failed because the control process exited with error code. See "systemctl status sshd.service" and "journalctl -xe" for details. [root@localhost ~]# journalctl -xe -- Defined-By: systemd -- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel -- -- Unit sshd.service has finished shutting down. Nov 05 02:50:24 localhost.localdomain systemd[1]: Starting OpenSSH server daemon -- Subject: Unit sshd.service has begun start-up -- Defined-By: systemd -- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel -- -- Unit sshd.service has begun starting up. Nov 05 02:50:24 localhost.localdomain sshd[1108]: /etc/ssh/sshd_config: Permissi Nov 05 02:50:24 localhost.localdomain systemd[1]: sshd.service: main process exi Nov 05 02:50:24 localhost.localdomain systemd[1]: Failed to start OpenSSH server -- Subject: Unit sshd.service has failed -- Defined-By: systemd -- Support: http://lists.freedesktop.org/mailman/listinfo/systemd-devel -- -- Unit sshd.service has failed. -- -- The result is failed. Nov 05 02:50:24 localhost.localdomain systemd[1]: Unit sshd.service entered fail Nov 05 02:50:24 localhost.localdomain systemd[1]: sshd.service failed. Nov 05 02:50:24 localhost.localdomain polkitd[732]: Unregistered Authentication [root@localhost ~]# cat /var/log/secure Nov 5 02:43:46 localhost polkitd[732]: Loading rules from directory /etc/polkit-1/rules.d Nov 5 02:43:46 localhost polkitd[732]: Loading rules from directory /usr/share/polkit-1/rules.d Nov 5 02:43:46 localhost polkitd[732]: Finished loading, compiling and executing 2 rules Nov 5 02:43:46 localhost polkitd[732]: Acquired the name org.freedesktop.PolicyKit1 on the system bus Nov 5 02:44:42 localhost login: pam_unix(login:session): session opened for user root by LOGIN(uid=0) Nov 5 02:44:42 localhost login: pam_lastlog(login:session): unable to open /var/log/lastlog: No such file or directory Nov 5 02:44:42 localhost login: ROOT LOGIN ON ttyAMA0 Nov 5 02:45:03 localhost polkitd[732]: Registered Authentication Agent for unix-process:924:7943 (system bus name :1.11 [/usr/bin/pkttyagent --notify-fd 5 --fallback], object path /org/freedesktop/PolicyKit1/AuthenticationAgent, locale en_US.UTF-8) Nov 5 02:46:36 localhost polkitd[732]: Unregistered Authentication Agent for unix-process:924:7943 (system bus name :1.11, object path /org/freedesktop/PolicyKit1/AuthenticationAgent, locale en_US.UTF-8) (disconnected from bus) Nov 5 02:48:25 localhost sshd[943]: Server listening on 0.0.0.0 port 22. Nov 5 02:48:25 localhost sshd[943]: Server listening on :: port 22. Nov 5 02:50:24 localhost polkitd[732]: Registered Authentication Agent for unix-process:1102:40050 (system bus name :1.17 [/usr/bin/pkttyagent --notify-fd 5 --fallback], object path /org/freedesktop/PolicyKit1/AuthenticationAgent, locale en_US.UTF-8) Nov 5 02:50:24 localhost polkitd[732]: Unregistered Authentication Agent for unix-process:1102:40050 (system bus name :1.17, object path /org/freedesktop/PolicyKit1/AuthenticationAgent, locale en_US.UTF-8) (disconnected from bus) [root@localhost ~]#
關閉selinux 沒有用
[root@localhost ~]# getenforce
Enforcing
[root@localhost ~]# cat /etc/sysconfig/selinux
sshd問題解決
Guest start: user@ovs:~$ /opt/qemu.git/bin/qemu-system-aarch64
-machine virt -cpu cortex-a57 -nographic -smp 1 -m 512
-kernel vmlinuz-run
-initrd initrd-run.img -append "root=/dev/sda2 console=ttyAMA0"
-global virtio-blk-device.scsi=off -device virtio-scsi-device,id=scsi
-drive file=debian8-arm64.img,id=rootimg,cache=unsafe,if=none -device scsi-hd,drive=rootimg -netdev user,id=unet
-device virtio-net-device,netdev=unet -net user,hostfwd=tcp:127.0.0.1:1122-:22
I think this is not a bug, but you are using the command line parameters in a wrong way.
When you use "-net user,hostfwd=tcp:127.0.0.1:1122-:22" you are creating a *new*,
second host network device which is not connected to the guest NIC device that you specified.
Please try to avoid mixing "-net" and "-netdev" options. You should rather do something like this instead: -netdev user,id=unet,hostfwd=tcp:127.0.0.1:1122-:22 -device virtio-net-device,netdev=unet
啟動命令
qemu-system-aarch64 -name vm2 -nographic \ -enable-kvm -M virt -cpu host -smp 2 -m 4096 \ -global virtio-blk-device.scsi=off \ -device virtio-scsi-device,id=scsi \ -kernel vmlinuz-4.18 --append "console=ttyAMA0 root=UUID=6a09973e-e8fd-4a6d-a8c0-1deb9556f477" \ -initrd initramfs-4.18 \ -drive file=vhuser-test1.qcow2 \ -netdev user,id=unet,hostfwd=tcp:127.0.0.1:1122-:22 -device virtio-net-device,netdev=unet \ -vnc :10 #./build/vhost-switch -l 0-3 -n 4 --huge-dir /dev/hugepages --socket-mem 1024 --log-level 8 -w 0000:07:00.1 -- --socket-file /tmp/vhost-user1 --client -p 0x1 --stats 20 #-chardev socket,id=char0,path=/tmp/vhost-user1,server \ #-netdev type=vhost-user,id=netdev0,chardev=char0,vhostforce \ #-device virtio-net-pci,netdev=netdev0,mac=52:54:00:00:00:01,mrg_rxbuf=on,rx_queue_size=1024,tx_queue_size=1024 \
sshd啟動成功
ssh登陸成功
qemu-system-aarch64: Failed to read msg header. Read 0 instead of 12. Original request 11. qemu-system-aarch64: vhost VQ 0 ring restore failed: -1: Resource temporarily unavailable (11) qemu-system-aarch64: Failed to set msg fds. qemu-system-aarch64: vhost VQ 1 ring restore failed: -1: Resource temporarily unavailable (11)
daemonize
qemu-system-aarch64 -name vm2 -daemonize \ -enable-kvm -M virt -cpu host -smp 2 -m 4096 \ -global virtio-blk-device.scsi=off \ -device virtio-scsi-device,id=scsi \ -kernel vmlinuz-4.18 --append "console=ttyAMA0 root=UUID=6a09973e-e8fd-4a6d-a8c0-1deb9556f477" \ -initrd initramfs-4.18 \ -drive file=vhuser-test1.qcow2 \ -netdev user,id=unet,hostfwd=tcp:127.0.0.1:1122-:22 -device virtio-net-device,netdev=unet \ -vnc :10
