Error: P4Runtime RPC error (INTERNAL): 'bf_pal_device_add(dev_id, &device_profile)' failed with error message: Internal error.

[RogueCHU]

I created 3 vhost-users and VMs, and I tried to add p4 pipeline binary package to the running IPDK container infrap4d switch. But it failed.

root@f7f2e82ffdeb:~# p4rt-ctl set-pipe br0 /root/examples/simple_l3/simple_l3.pb.bin /root/examples/simple_l3/p4Info.txt
Error: P4Runtime RPC error (INTERNAL): 'bf_pal_device_add(dev_id, &device_profile)' failed with error message: Internal error. 

how I created three vhost-user-x ports in ipdk container

root@f7f2e82ffdeb:~#gnmi-ctl set "device:virtual-device,name:net_vhost0,host-name:host1,device-type:VIRTIO_NET,queues:1,socket-path:/tmp/intf/vhost-user-0,port-type:LINK"

root@f7f2e82ffdeb:~#gnmi-ctl set "device:virtual-device,name:net_vhost1,host-name:host1,device-type:VIRTIO_NET,queues:1,socket-path:/tmp/intf/vhost-user-1,port-type:LINK"

root@f7f2e82ffdeb:~#gnmi-ctl set "device:virtual-device,name:net_vhost2,host-name:host1,device-type:VIRTIO_NET,queues:1,socket-path:/tmp/intf/vhost-user-2,port-type:LINK"

how I created three VMs in host

export WORKING_DIR=/root/.ipdk/volume
#vm1 start
qemu-system-x86_64 -smp 1 -m 256M \
    -boot c -cpu host --enable-kvm -nographic \
    -name VM1 \
    -hda "${WORKING_DIR}"/images/vm1.qcow2 \
    -drive file="${WORKING_DIR}"/images/seed1.img,id=seed,if=none,format=raw,index=1 \
    -device virtio-blk,drive=seed \
    -object memory-backend-file,id=mem,size=256M,mem-path=/mnt/huge,share=on \
    -numa node,memdev=mem \
    -mem-prealloc \
    -chardev socket,id=char1,path="${WORKING_DIR}"/intf/vhost-user-0 \
    -netdev type=vhost-user,id=netdev0,chardev=char1,vhostforce \
    -device virtio-net-pci,mac=52:54:00:34:12:aa,netdev=netdev0 \
    -serial telnet::6551,server,nowait &

#vm2 start
qemu-system-x86_64 -smp 1 -m 256M \
    -boot c -cpu host --enable-kvm -nographic \
    -name VM2 \
    -hda "${WORKING_DIR}"/images/vm2.qcow2 \
    -drive file="${WORKING_DIR}"/images/seed2.img,id=seed,if=none,format=raw,index=1 \
    -device virtio-blk,drive=seed \
    -object memory-backend-file,id=mem,size=256M,mem-path=/mnt/huge,share=on \
    -numa node,memdev=mem \
    -mem-prealloc \
    -chardev socket,id=char2,path="${WORKING_DIR}"/intf/vhost-user-1 \
    -netdev type=vhost-user,id=netdev1,chardev=char2,vhostforce \
    -device virtio-net-pci,mac=52:54:00:34:12:bb,netdev=netdev1 \
    -serial telnet::6552,server,nowait

#vm3 start
qemu-system-x86_64 -smp 1 -m 256M \
    -boot c -cpu host --enable-kvm -nographic \
    -name VM3 \
    -hda /root/.ipdk/volume/images/focal-server-cloudimg-amd64.img \
    -drive file="${WORKING_DIR}"/images/seed3.img,id=seed,if=none,format=raw,index=1 \
    -device virtio-blk,drive=seed \
    -object memory-backend-file,id=mem,size=256M,mem-path=/mnt/huge,share=on \
    -numa node,memdev=mem \
    -mem-prealloc \
    -chardev socket,id=char3,path="${WORKING_DIR}"/intf/vhost-user-2 \
    -netdev type=vhost-user,id=netdev2,chardev=char3,vhostforce \
    -device virtio-net-pci,mac=52:54:00:34:12:cc,netdev=netdev2 \
    -serial telnet::6553,server,nowait

my p4 program was simple_l3.p4

/* -*- P4_16 -*- */

#include <core.p4>
#include <psa.p4>

/*************************************************************************
 ************* C O N S T A N T S    A N D   T Y P E S  *******************
**************************************************************************/
const bit<16> ETHERTYPE_TPID = 0x8100;
const bit<16> ETHERTYPE_IPV4 = 0x0800;

/*************************************************************************
 ***********************  H E A D E R S  *********************************
 *************************************************************************/

/*  Define all the headers the program will recognize             */
/*  The actual sets of headers processed by each gress can differ */

/* Standard ethernet header */
header ethernet_h {
    bit<48>   dst_addr;
    bit<48>   src_addr;
    bit<16>   ether_type;
}

header vlan_tag_h {
    bit<16> pcp_cfi_vid;
    bit<16>  ether_type;
}

header ipv4_h {
    bit<8>       version_ihl;
    bit<8>       diffserv;
    bit<16>      total_len;
    bit<16>      identification;
    bit<16>      flags_frag_offset;
    bit<8>       ttl;
    bit<8>       protocol;
    bit<16>      hdr_checksum;
    bit<32>      src_addr;
    bit<32>      dst_addr;
}

const int IPV4_HOST_SIZE = 65536;

typedef bit<48> ethernet_addr_t;

/*************************************************************************
 **************  I N G R E S S   P R O C E S S I N G   *******************
 *************************************************************************/

    /***********************  H E A D E R S  ************************/

struct my_ingress_headers_t {
    ethernet_h   ethernet;
    vlan_tag_h   vlan_tag;
    ipv4_h       ipv4;
}

    /******  G L O B A L   I N G R E S S   M E T A D A T A  *********/

struct my_ingress_metadata_t {
}

struct empty_metadata_t {
}

    /***********************  P A R S E R  **************************/
parser Ingress_Parser(
    packet_in pkt,
    out my_ingress_headers_t hdr,
    inout my_ingress_metadata_t meta,
    in psa_ingress_parser_input_metadata_t ig_intr_md,
    in empty_metadata_t resub_meta, 
    in empty_metadata_t recirc_meta)
{
     state start {
        transition parse_ethernet;
    }

    state parse_ethernet {
        pkt.extract(hdr.ethernet);
        transition select(hdr.ethernet.ether_type) {
            ETHERTYPE_TPID:  parse_vlan_tag;
            ETHERTYPE_IPV4:  parse_ipv4;
            default: accept;
        }
    }

    state parse_vlan_tag {
        pkt.extract(hdr.vlan_tag);
        transition select(hdr.vlan_tag.ether_type) {
            ETHERTYPE_IPV4:  parse_ipv4;
            default: accept;
        }
    }

    state parse_ipv4 {
        pkt.extract(hdr.ipv4);
        transition accept;
    }

}

    /***************** M A T C H - A C T I O N  *********************/

control ingress(
    inout my_ingress_headers_t hdr,
    inout my_ingress_metadata_t meta,
    in psa_ingress_input_metadata_t ig_intr_md,
    inout psa_ingress_output_metadata_t ostd
)
{
    action send(PortId_t port) {
        send_to_port(ostd, port);
    }

    action drop() {
        ingress_drop(ostd);
    }
    
    table ipv4_host {
        key = { hdr.ipv4.dst_addr : exact; }
        actions = {
            send;drop;
            @defaultonly NoAction;
        }

        const default_action = drop();

        size = IPV4_HOST_SIZE;
    }


    apply {
                ipv4_host.apply();
    }
}

    /*********************  D E P A R S E R  ************************/

control Ingress_Deparser(packet_out pkt,
    out empty_metadata_t clone_i2e_meta, 
    out empty_metadata_t resubmit_meta, 
    out empty_metadata_t normal_meta,
    inout my_ingress_headers_t hdr,
    in    my_ingress_metadata_t meta,
    in psa_ingress_output_metadata_t istd)
{
    apply {
        pkt.emit(hdr);
    }
}


/*************************************************************************
 ****************  E G R E S S   P R O C E S S I N G   *******************
 *************************************************************************/

    /***********************  H E A D E R S  ************************/

struct my_egress_headers_t {
}

    /********  G L O B A L   E G R E S S   M E T A D A T A  *********/

struct my_egress_metadata_t {
}

    /***********************  P A R S E R  **************************/

parser Egress_Parser(
    packet_in pkt,
    out my_egress_headers_t hdr,
    inout my_ingress_metadata_t meta,
    in psa_egress_parser_input_metadata_t istd, 
    in empty_metadata_t normal_meta, 
    in empty_metadata_t clone_i2e_meta, 
    in empty_metadata_t clone_e2e_meta)
{
    state start {
        transition accept;
    }
}

    /***************** M A T C H - A C T I O N  *********************/

control egress(
    inout my_egress_headers_t hdr,
    inout my_ingress_metadata_t meta,
    in psa_egress_input_metadata_t istd, 
    inout psa_egress_output_metadata_t ostd)
{
    apply {
    }
}

    /*********************  D E P A R S E R  ************************/

control Egress_Deparser(packet_out pkt,
    out empty_metadata_t clone_e2e_meta, 
    out empty_metadata_t recirculate_meta,
    inout my_egress_headers_t hdr,
    in my_ingress_metadata_t meta,
    in psa_egress_output_metadata_t istd, 
    in psa_egress_deparser_input_metadata_t edstd)
{
    apply {
        pkt.emit(hdr);
    }
}

#if __p4c__
bit<32> test_version = __p4c_version__;
#endif

/************ F I N A L   P A C K A G E ******************************/

IngressPipeline(Ingress_Parser(), ingress(), Ingress_Deparser()) pipe;

EgressPipeline(Egress_Parser(), egress(), Egress_Deparser()) ep;

PSA_Switch(pipe, PacketReplicationEngine(), ep, BufferingQueueingEngine()) main;

any idea? thank you very much guys!

[jafingerhut]

It might be possible that some recent fixes to p4c-dpdk back end fix some of these issues.

I have tried the latest version as of 2023-Sep-14 of p4c-dpdk on your P4 program, with a 1-month old version of the IPDK networking container, and there is no error in loading the code into the P4-DPDK software switch.

Note: I chose to compile the latest version of p4c-dpdk outside of the networking container in my testing, because I am not familiar with how to change the ipdk build to cause it to compile a particular version.

[RogueCHU]

It might be possible that some recent fixes to p4c-dpdk back end fix some of these issues.

I have tried the latest version as of 2023-Sep-14 of p4c-dpdk on your P4 program, with a 1-month old version of the IPDK networking container, and there is no error in loading the code into the P4-DPDK software switch.

Note: I chose to compile the latest version of p4c-dpdk outside of the networking container in my testing, because I am not familiar with how to change the ipdk build to cause it to compile a particular version.

Thank you for your reply! But I still got the same error after trying p4c-dpdk. It's worth mentioning that the program ran successfully when I used two vhost-user ports. But when I created three vhost-user ports, the above error occured.

Does this mean that P4-DPDK soft switch is not compatible with three vhost-user ports? Or what changes should I make to the simple_l3.p4 program?

Thank you!

[RogueCHU]

By the way, here were the error details in p4driver.log.

2023-09-15 10:08:42.438831 INFO  BF_PAL Exiting fixed_function_dev_add
2023-09-15 10:08:42.438837 INFO  BF_PORT Entering port_mgr_dev_add
2023-09-15 10:08:42.438843 INFO  BF_PORT Enter port_mgr_config_import
2023-09-15 10:08:42.438848 INFO  BF_PORT port_mgr_config_import: device 0, file (null)
2023-09-15 10:08:42.438855 INFO  BF_PORT Exit port_mgr_config_import
2023-09-15 10:08:42.708376 ERROR BF_PIPE dal_enable_pipeline line:158  Pipeline build failed

2023-09-15 10:08:42.708467 ERROR BF_PIPE Failed to Build Pipeline pipe
2023-09-15 10:08:42.708493 ERROR BF_DVM Device add handling failed for dev 0, sts Internal error (22), Client pipe-mgr 
2023-09-15 10:08:42.708509 ERROR BF_DVM Device add failed for dev 0, sts Internal error (22)
2023-09-15 10:08:42.708537 INFO  BF_PORT Entering port_mgr_dev_remove
2023-09-15 10:08:42.708545 INFO  BF_PAL STUB fixed_function_dev_remove
2023-09-15 10:08:42.708552 INFO  BF_LLD Entering dummy STUB:lld_dev_remove

2023-09-15 10:08:42.708558 INFO  BF_LLD Exiting dummy STUB:lld_dev_remove

Any suggestions of how to solve this? Thank you very much.

[jafingerhut]

I have learned that there is a restriction that things do not work unless you create a number of ports that is a power of 2. So 3 will fail, but 4 should work again.

[jafingerhut]

@5abeel @ffoulkes Do either of you know if it is straightforward to change the code so that it could show a very clear error message when this occurs, preferably both to stdout/stderr, and in the infrap4d log? Perhaps a message that included a sentence like "Failing to start with 3 interfaces. P4-DPDK requires that the number of interfaces be a power of 2, e.g. use 4 interfaces instead of 3."

Without such a clear message, it is a complete mystery why such failures occur right now.

[RogueCHU]

I have learned that there is a restriction that things do not work unless you create a number of ports that is a power of 2. So 3 will fail, but 4 should work again.

Wow! As you said, it worked! Thanks a lot.
And I also want to know why this error occurs. Hope someone can give an answer.
Thank you again.

[jafingerhut]

I have heard that the error occurs because there is some part of the P4-DPDK implementation that checks whether the number of ports is a power of 2, and if it is not, it returns with an error. It makes this check because it does not support non-power-of-2 number of ports, because of some part of the implementation where it needs to calculate some load balance roughly equally over each port, and wants to simplify the calculations involved to simple bit masking, rather than doing a modulo by a non-power-of-2 number. But this is hearsay -- I have not seen the lines of code involved.

[5abeel]

@5abeel @ffoulkes Do either of you know if it is straightforward to change the code so that it could show a very clear error message when this occurs, preferably both to stdout/stderr, and in the infrap4d log? Perhaps a message that included a sentence like "Failing to start with 3 interfaces. P4-DPDK requires that the number of interfaces be a power of 2, e.g. use 4 interfaces instead of 3."

Without such a clear message, it is a complete mystery why such failures occur right now.

Since this is a limitation from DPDK, perhaps it makes more sense to add an error response from SDE layer? I can start an offline discussion with you and someone from SDE team to find the right spot for this improvement.

[cristian-dumitrescu]

We need to remove this limitation and take the small performance hit. I will put it on my to-do list.
Meanwhile, the workaround is to work with 4 input ports instead of 3.

[5abeel]

@satish153 to check into this issue

[ffoulkes]

I tried to locate the error message in the source code and failed. I don't know where it's coming from.

Ideally, we should detect the error when the number of interfaces is defined. It's an invalid input.

[satish153]

I tried to locate the error message in the source code and failed. I don't know where it's coming from.

Ideally, we should detect the error when the number of interfaces is defined. It's an invalid input.

https://github.com/DPDK/dpdk/blob/main/lib/pipeline/rte_swx_pipeline.c#L342

[ffoulkes]

I tried to locate the error message in the source code and failed. I don't know where it's coming from.
Ideally, we should detect the error when the number of interfaces is defined. It's an invalid input.

https://github.com/DPDK/dpdk/blob/main/lib/pipeline/rte_swx_pipeline.c#L342

Ouch. No wonder I didn't find it.