Using the Radxa Rock Pi N10 NPU on Mainline Linux
So you ditched the old v4.4 Rockchip vendor kernel in favor of Mainline Linux and now half of your SBC doesn't work! Well, trouble no more, at least I'll help you get a working NPU for accelerated AI inference.
First, you will need to download the latest NPU firmware and matching rknn-api from this official Rockchip repository.
Second, you will need to manually apply the following patch to your Mainline Linux kernel and build it. I should have probably reworked it a bit to not modify rk3399.dtsi and rk3399pro-vmarc-som.dtsi but instead modify only rk3399pro-rock-pi-n10.dts. Also, the PCIe bits are unimportant since the Rock Pi N10 RK3399Pro NPU talks to the CPU over USB, USB2 when in maskrom mode then USB3 after we push the firmware with our modified npu_upgrade script. The RTC commenting out of a check may be uneeded, but make sure you enable the clk modifications. Beware, messing out with your clocks via sysfs without knowing what you are doing may permanently damage your device at the hardware level, so if you're clueless, don't mess around. Most of the commands in this guide require root privileges.
From: Geraldo Nascimento <[email protected]>
Date: Thu, 31 Aug 2023 01:24:38 -0300
Subject: [PATCH 9/9] Activate weird options for NPU - Dangerous!
---
arch/arm64/boot/dts/rockchip/rk3399.dtsi | 4 +-
.../dts/rockchip/rk3399pro-vmarc-som.dtsi | 52 +++++++++++++++++--
drivers/clk/clk.c | 1 +
drivers/rtc/rtc-hym8563.c | 4 +-
4 files changed, 52 insertions(+), 9 deletions(-)
diff --git a/arch/arm64/boot/dts/rockchip/rk3399.dtsi b/arch/arm64/boot/dts/rockchip/rk3399.dtsi
index be0838c85563..581081ba3698 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3399.dtsi
@@ -489,7 +489,7 @@ usbdrd_dwc3_1: usb@fe900000 {
clocks = <&cru SCLK_USB3OTG1_REF>, <&cru ACLK_USB3OTG1>,
<&cru SCLK_USB3OTG1_SUSPEND>;
clock-names = "ref", "bus_early", "suspend";
- dr_mode = "otg";
+ dr_mode = "host";
phys = <&u2phy1_otg>, <&tcphy1_usb3>;
phy-names = "usb2-phy", "usb3-phy";
phy_type = "utmi_wide";
@@ -2897,7 +2897,7 @@ pcie_clkreqn_cpm: pci-clkreqn-cpm {
pcie_clkreqnb_cpm: pci-clkreqnb-cpm {
rockchip,pins =
- <4 RK_PD0 RK_FUNC_GPIO &pcfg_pull_none>;
+ <4 RK_PD0 1 &pcfg_pull_none>;
};
};
diff --git a/arch/arm64/boot/dts/rockchip/rk3399pro-vmarc-som.dtsi b/arch/arm64/boot/dts/rockchip/rk3399pro-vmarc-som.dtsi
index c487dcb46ad4..58bc25a18953 100644
--- a/arch/arm64/boot/dts/rockchip/rk3399pro-vmarc-som.dtsi
+++ b/arch/arm64/boot/dts/rockchip/rk3399pro-vmarc-som.dtsi
@@ -105,7 +105,7 @@ rk809: pmic@20 {
#clock-cells = <1>;
clock-output-names = "rk808-clkout1", "rk808-clkout2";
pinctrl-names = "default";
- pinctrl-0 = <&pmic_int_l>;
+ pinctrl-0 = <&pmic_int_l>, <&clk_32k>;
rockchip,system-power-controller;
wakeup-source;
@@ -329,10 +329,10 @@ &i2c2 {
hym8563: rtc@51 {
compatible = "haoyu,hym8563";
reg = <0x51>;
+ pinctrl-names = "default";
+ pinctrl-0 = <&hym8563_int>, <&npu_ref_clk>;
#clock-cells = <0>;
clock-output-names = "hym8563";
- pinctrl-names = "default";
- pinctrl-0 = <&hym8563_int>;
interrupt-parent = <&gpio4>;
interrupts = <RK_PD6 IRQ_TYPE_LEVEL_LOW>;
};
@@ -385,7 +385,7 @@ &pcie_phy {
&pcie0 {
ep-gpios = <&gpio0 RK_PB4 GPIO_ACTIVE_HIGH>;
num-lanes = <4>;
- pinctrl-0 = <&pcie_clkreqnb_cpm>;
+ pinctrl-0 = <&pcie_clkreqnb_cpm>, <&pcie_perst>;
pinctrl-names = "default";
vpcie0v9-supply = <&vcca_0v9>; /* VCC_0V9_S0 */
vpcie1v8-supply = <&vcca_1v8>; /* VCC_1V8_S0 */
@@ -394,6 +394,7 @@ &pcie0 {
};
&pinctrl {
+
hym8563 {
hym8563_int: hym8563-int {
rockchip,pins = <4 RK_PD6 0 &pcfg_pull_up>;
@@ -410,6 +411,10 @@ pcie {
pcie_pwr: pcie-pwr {
rockchip,pins = <4 RK_PD4 RK_FUNC_GPIO &pcfg_pull_up>;
};
+
+ pcie_perst: pcie-perst {
+ rockchip,pins = <0 RK_PB4 RK_FUNC_GPIO &pcfg_pull_none>;
+ };
};
pmic {
@@ -435,6 +440,12 @@ usb0_en_oc: usb0-en-oc {
rockchip,pins = <4 RK_PD2 RK_FUNC_GPIO &pcfg_pull_up>;
};
};
+
+ npu_clk {
+ npu_ref_clk: npu-ref-clk {
+ rockchip,pins = <0 RK_PA2 1 &pcfg_pull_none>;
+ };
+ };
};
&pmu_io_domains {
@@ -469,6 +480,10 @@ &tcphy0 {
status = "okay";
};
+&tcphy1 {
+ status = "okay";
+};
+
&tsadc {
rockchip,hw-tshut-mode = <1>;
rockchip,hw-tshut-polarity = <1>;
@@ -489,10 +504,14 @@ u2phy0_host: host-port {
};
};
-
&u2phy1 {
status = "okay";
+ u2phy1_otg: otg-port {
+ phy-supply = <&vbus_typec>;
+ status = "okay";
+ };
+
u2phy1_host: host-port {
phy-supply = <&vbus_host>;
status = "okay";
@@ -519,10 +538,18 @@ &usbdrd3_0 {
status = "okay";
};
+&usbdrd3_1 {
+ status = "okay";
+};
+
&usbdrd_dwc3_0 {
status = "okay";
};
+&usbdrd_dwc3_1 {
+ status = "okay";
+};
+
&gpu {
status = "okay";
};
@@ -531,6 +558,21 @@ &hdmi_sound {
status = "okay";
};
+/*&spi1 {
+ status = "okay";
+
+ flash@0 {
+ #address-cells = <1>;
+ #size-cells = <1>;
+ compatible = "jedec,spi-nor";
+ reg = <0x0>;
+ spi-max-frequency = <40000000>;
+ spi-cpha;
+ spi-cpol;
+ status = "okay";
+ };
+};*/
+
&vbus_host {
enable-active-high;
gpio = <&gpio4 RK_PD1 GPIO_ACTIVE_HIGH>; /* USB1_EN_OC# */
diff --git a/drivers/clk/clk.c b/drivers/clk/clk.c
index c249f9791ae8..71ce5c69849c 100644
--- a/drivers/clk/clk.c
+++ b/drivers/clk/clk.c
@@ -3309,6 +3309,7 @@ static int clk_dump_show(struct seq_file *s, void *data)
DEFINE_SHOW_ATTRIBUTE(clk_dump);
#undef CLOCK_ALLOW_WRITE_DEBUGFS
+#define CLOCK_ALLOW_WRITE_DEBUGFS
#ifdef CLOCK_ALLOW_WRITE_DEBUGFS
/*
* This can be dangerous, therefore don't provide any real compile time
diff --git a/drivers/rtc/rtc-hym8563.c b/drivers/rtc/rtc-hym8563.c
index b018535c842b..c14c54a3c4bf 100644
--- a/drivers/rtc/rtc-hym8563.c
+++ b/drivers/rtc/rtc-hym8563.c
@@ -97,11 +97,11 @@ static int hym8563_rtc_read_time(struct device *dev, struct rtc_time *tm)
if (ret < 0)
return ret;
- if (buf[0] & HYM8563_SEC_VL) {
+ /*if (buf[0] & HYM8563_SEC_VL) {
+ dev_warn(&client->dev,
"no valid clock/calendar values available\n");
return -EINVAL;
- }
+ }*/
tm->tm_sec = bcd2bin(buf[0] & HYM8563_SEC_MASK);
tm->tm_min = bcd2bin(buf[1] & HYM8563_MIN_MASK);
--
2.39.0That's it for the Mainline Linux kernel/dtb parts.
Once you boot with your modified kernel and dtb, set the date with either the date command by running for example date 083116202023, that'll set the date to Thu Aug 31 16:20:00 -03 2023 or optionally do it via ntp. Just make sure your clock is set to current time. Then run hwclock --systohc to write the correct time to the RTC. Don't skip this step.
Moving on, you will need a working copy of libgpiod. Either compíle it yourself or get it from your distro. I compiled my own so I will cd to libgpiod/tools and run the following script, which you can call rk3399pro_npu_powerctl.sh for example:
echo "1" > /sys/kernel/debug/clk/rk808-clkout2/clk_prepare_enable
echo "24000000" > /sys/kernel/debug/clk/clk_wifi_pmu/clk_rate
#echo "0" > /sys/kernel/debug/clk/clk_wifi_pmu/clk_prepare_enable
./gpioset -zc 0 10=0
sleep 0.2000
./gpioset -zc 1 22=0
./gpioset -zc 1 23=0
./gpioset -zc 1 24=0
./gpioset -zc 0 11=0
./gpioset -zc 0 4=0
./gpioset -zc 1 3=0
./gpioset -zc 1 0=0
sleep 0.2000
killall -15 gpioset
./gpioset -zc 0 4=1
sleep 0.2000
./gpioset -zc 0 10=1
sleep 0.2000
./gpioset -zc 0 11=1
sleep 0.2000
echo "1" > /sys/kernel/debug/clk/clk_wifi_pmu/clk_prepare_enable
./gpioset -zc 1 22=1
sleep 0.2000
./gpioset -zc 1 23=1
sleep 0.2000
./gpioset -zc 1 24=1
sleep 2.5000
./gpioset -zc 1 0=1This script mostly replicates the functionality of npu_powerctrl binary shipped by Rockchip / Radxa, except with libgpiod gpioset tool instead of deprecated sysfs gpio manipulation.
A point of note: the NPU requires a 24MHz clock or it won't work - it definitely doesn't work with the default WiFi 26MHz clock. I have no idea what happens if you're using the Radxa WiFi module together with this script, so again, beware.
After running the script you should see something similar to this in your dmesg:
[ 119.665991] usb 3-1: new high-speed USB device number 2 using xhci-hcd
[ 119.792904] usb 3-1: New USB device found, idVendor=2207, idProduct=180a, bcdDevice= 1.00
[ 119.792930] usb 3-1: New USB device strings: Mfr=0, Product=0, SerialNumber=0This means the NPU is active and on Maskrom mode, ready to receive a firmware update.
Next, we will execute the following modified npu_upgrade script. It's a little rough around the edges and it depends on a Rockchip binary called upgrade_tool which I have extracted from the official Radxa image in case you don't want to bother with that - just rename from upgrade_tool.bin to upgrade_tool. Make sure to modify the paths at the script header to match your local machine paths. Usage should be ./npu_upgrade MiniLoaderAll.bin uboot.img trust.img boot.img:
#!/bin/bash
PROGRAM=${0##*/}
#reset npu
#/usr/bin/npu_powerctrl -i
#/usr/bin/npu_powerctrl -o
sleep 1
if [ $# -ne 4 ]; then
echo 'Usage: '$PROGRAM' loader uboot trust boot'
exit
fi
DIR="/pathTo/rknpu/drivers/npu_firmware/npu_fw"
UPGRADE_TOOL=/pathTo/upgrade_tool
LOADER=$DIR/$1
UBOOT=$DIR/$2
TRUST=$DIR/$3
BOOT=$DIR/$4
UBOOT_ADDR=0x40000
TRUST_ADDR=0x40800
BOOT_ADDR=0x20000
function download_func()
{
local RET1=1
echo 'start to download loader...' >> /tmp/npu.log
$UPGRADE_TOOL db $LOADER > /dev/null
if [ $? -ne 0 ]; then
echo 'failed to download loader!' >> /tmp/npu.log
return $RET1;
fi
echo 'download loader ok' >> /tmp/npu.log
sleep 1
echo 'start to wait loader...' >> /tmp/npu.log
$UPGRADE_TOOL td > /dev/null
if [ $? -ne 0 ]; then
echo 'failed to wait loader!' >> /tmp/npu.log
return $RET1
fi
echo 'loader is ready' >> /tmp/npu.log
echo 'start to write uboot...' >> /tmp/npu.log
$UPGRADE_TOOL wl $UBOOT_ADDR $UBOOT > /dev/null
if [ $? -ne 0 ]; then
echo 'failed to write uboot!' >> /tmp/npu.log
return $RET1
fi
echo 'write uboot ok' >> /tmp/npu.log
echo 'start to write trust...' >> /tmp/npu.log
$UPGRADE_TOOL wl $TRUST_ADDR $TRUST > /dev/null
if [ $? -ne 0 ]; then
echo 'failed to write trust!' >> /tmp/npu.log
return $RET1
fi
echo 'write trust ok' >> /tmp/npu.log
echo 'start to write boot...' >> /tmp/npu.log
$UPGRADE_TOOL wl $BOOT_ADDR $BOOT > /dev/null
if [ $? -ne 0 ]; then
echo 'failed to write boot!' >> /tmp/npu.log
return $RET1
fi
echo 'write boot ok' >> /tmp/npu.log
RET1=0
return $RET1
}
function check_device_ready_func()
{
echo 'start to wait device...' > /tmp/npu.log
local i=0
local RET=1
while [ $i -lt 10 ]; do
$UPGRADE_TOOL ld > /dev/null
if [ $? -ne 0 ]; then
i=$((i+1))
echo $i
sleep 0.1
else
sleep 0.1
break
fi
if [ $i -eq 5 ]; then
/usr/bin/npu_powerctrl -o
sleep 3
echo 'reset npu to retry!!!' >> /tmp/npu.log
fi
done
if [ $i -ge 10 ]; then
echo 'failed to wait device!' >> /tmp/npu.log
return $RET
fi
echo 'device is ready' >> /tmp/npu.log
RET=0
return $RET
}
function poweron_Npu_func()
{
/usr/bin/npu_powerctrl -i
sleep 0.1
/usr/bin/npu_powerctrl -o
sleep 1
}
if [ ! -f $UPGRADE_TOOL ]; then
echo $UPGRADE_TOOL 'is not existed!'
exit
fi
if [ ! -f $LOADER ]; then
echo $LOADER 'is not existed!'
exit
fi
if [ ! -f $UBOOT ]; then
echo $UBOOT 'is not existed!'
exit
fi
if [ ! -f $TRUST ]; then
echo $TRUST 'is not existed!'
exit
fi
if [ ! -f $BOOT ]; then
echo $BOOT 'is not existed!'
exit
fi
check_device_ready_func
if [ $? = 1 ];then
echo "check_device_ready error!!!" >> /tmp/npu.log
poweron_Npu_func
check_device_ready_func
fi
download_func
if [ $? = 1 ];then
echo "reset download_func"
echo 'down load error!!! reset usb hub' >> /tmp/npu.log
#echo 0 > /sys/class/gpio/gpio149/value
#sleep 3
#echo 1 > /sys/class/gpio/gpio149/value
#poweron_Npu_func
#check_device_ready_func
#download_func
fi
echo 'start to run system...' >> /tmp/npu.log
$UPGRADE_TOOL rs $UBOOT_ADDR $TRUST_ADDR $BOOT_ADDR $UBOOT $TRUST $BOOT > /dev/null
if [ $? -ne 0 ]; then
echo 'failed to run system!' >> /tmp/npu.log
exit
fi
echo 'run system ok' >> /tmp/npu.logIf it works you should see something similar to the following on your dmesg:
[ 129.886423] usb 3-1: reset high-speed USB device number 2 using xhci-hcd
[ 130.007927] usb 3-1: device descriptor read/64, error -71
[ 130.236951] usb 3-1: device firmware changed
[ 130.239958] usb 3-1: USB disconnect, device number 2
[ 130.354991] usb 3-1: new high-speed USB device number 3 using xhci-hcd
[ 130.483908] usb 3-1: New USB device found, idVendor=2207, idProduct=180a, bcdDevice= 1.00
[ 130.483933] usb 3-1: New USB device strings: Mfr=1, Product=2, SerialNumber=3
[ 130.483940] usb 3-1: Product: USB-MSC
[ 130.483946] usb 3-1: Manufacturer: RockChip
[ 130.483952] usb 3-1: SerialNumber: rockchip
[ 135.170210] usb 3-1: USB disconnect, device number 3
[ 137.859118] usb 4-1: new SuperSpeed USB device number 2 using xhci-hcd
[ 137.873716] usb 4-1: New USB device found, idVendor=2207, idProduct=0019, bcdDevice= 4.04
[ 137.873740] usb 4-1: New USB device strings: Mfr=1, Product=2, SerialNumber=3
[ 137.873747] usb 4-1: Product: rk3xxx
[ 137.873753] usb 4-1: Manufacturer: rockchip
[ 137.873805] usb 4-1: SerialNumber: 526aef7c692b0fedThis means it worked. npu_transfer_proxy devices should show attached NPU via USB3 and npu_tranfer_proxy will now work and the matching rknn-api C Demos should work.
Happy AI Hacking!