Fragment disappears after setAdapter on ListView

I have a ListView in a Fragment containing also some other UI-Elements. Everything works until I call setAdapter on the ListView. In that moment I can debug that the ListView is filled with the elements but immmediately after the whole fragment disappears (including the other UI-Elements). If I set the Adapter 0,1 seconds later, everything works.

public View onCreateView(LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) {
    mView = inflater.inflate(R.layout.fragment_listview, null);
    mListView = (ListView)mView.findViewById(R.id.list_items);

    TextView emptyView = (TextView)mView.findViewById(R.id.text_empty);
    mListView.setEmptyView(emptyView);

    final Handler handler = new Handler();
    handler.postDelayed(new Runnable() {
        @Override
        public void run() {
            // this is the only work around to not make the list view disappear
            loadContacts();
        }
    }, 100);
    // if I call loadContacts() here, the Fragment disappears

    mListView.setOnItemClickListener(new AdapterView.OnItemClickListener() {
        public void onItemClick(AdapterView<?> parent, View view, int position, long id) {
            Person p = mContacts.get(position);
            if (pickerInterface != null) {
                pickerInterface.pickedContact(p);
            }
        }
    });
    return mView;
}


public void loadContacts() {
    mContacts = ContactsDataSource.getAllPhoneContacts(getActivity(), true);
    mAdapter = new ContactsAdapter(getActivity());
    mAdapter.mContacts = mContacts;
    mAdapter.isPlainList = true;
    // after the following line the Fragment is filled and then immediately disappears
    mListView.setAdapter(mAdapter);
}

the ContactsAdapter is very simple:

public class ContactsAdapter extends BaseAdapter {
    public List<Person> mContacts = new ArrayList<Person>();
    public Activity context;

    // Constructor
    public ContactsAdapter(Activity c){
        context = c;
    }

    @Override
    public int getCount() {
        return mContacts.size();
    }

    public Person getItem(int position){
        return mContacts.get(position);
    }

    @Override
    public boolean isEmpty()
    {
        return mContacts.size() == 0;
    }

    @Override
    public long getItemId(int position) {
        return position;
    }

    @SuppressLint("InflateParams")
    public View getView(int position, View convertView, ViewGroup viewGroup) {
        if (convertView == null) {
            LayoutInflater inflater = (LayoutInflater)context.getSystemService(Context.LAYOUT_INFLATER_SERVICE);
            convertView = inflater.inflate(R.layout.list_item_contact, null);
        }
        Person person = mContacts.get(position);

        TextView tvName = (TextView) convertView.findViewById(R.id.text_name);
        tvName.setText(person.getDisplayName());
        return convertView;
    }
}

I have spent hours debugging and searching for reasons and have no clue. Thanks for any idea.

Edit: XML as following:

<?xml version="1.0" encoding="utf-8"?>
<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    android:background="@color/white" >

    <ListView
        android:id="@+id/list_items"
        android:layout_width="match_parent"
        android:layout_height="match_parent"
        android:layout_above="@+id/line_horizontal"
        android:layout_alignParentTop="true" />

    <TextView
        android:id="@+id/text_empty"
        android:layout_width="match_parent"
        android:layout_height="wrap_content"
        android:gravity="center"
        android:padding="15dp"
        android:layout_alignParentTop="true"
        android:text="@string/label_no_entries"
        android:textColor="@color/grey"
        android:textSize="21sp" />

    <View
        android:id="@+id/line_horizontal"
        android:layout_width="match_parent"
        android:layout_height="1dp"
        android:layout_above="@+id/layout_navigation"
        android:background="@color/black" />

    <RelativeLayout
        android:id="@+id/layout_navigation"
        android:layout_width="match_parent"
        android:layout_height="wrap_content"
        android:layout_alignParentBottom="true" >

        <TextView
            android:id="@+id/text_scrollback"
            android:layout_width="wrap_content"
            android:layout_height="wrap_content"
            android:layout_alignParentLeft="true"
            android:drawablePadding="10dp"
            android:drawableRight="@drawable/backward"
            android:gravity="center_vertical"
            android:paddingBottom="15dp"
            android:paddingLeft="10dp"
            android:paddingTop="15dp"
            android:text="@string/activity_contacts_backward_label"
            android:textColor="@color/mediumgrey" />

        <View
            android:id="@+id/view_coverback"
            android:layout_width="wrap_content"
            android:layout_height="wrap_content"
            android:layout_alignBottom="@+id/text_scrollback"
            android:layout_alignParentLeft="true"
            android:layout_alignRight="@+id/text_scrollback"
            android:layout_alignTop="@+id/text_scrollback"
            android:background="#c0ffffff" />

        <TextView
            android:id="@+id/text_scrollforward"
            android:layout_width="wrap_content"
            android:layout_height="wrap_content"
            android:layout_alignParentRight="true"
            android:drawableLeft="@drawable/forward"
            android:drawablePadding="10dp"
            android:gravity="center_vertical"
            android:paddingBottom="15dp"
            android:paddingRight="10dp"
            android:paddingTop="15dp"
            android:text="@string/activity_contacts_forward_label"
            android:textColor="@color/mediumgrey" />

        <View
            android:id="@+id/view_coverforward"
            android:layout_width="wrap_content"
            android:layout_height="wrap_content"
            android:layout_alignBottom="@+id/text_scrollforward"
            android:layout_alignParentRight="true"
            android:layout_alignLeft="@+id/text_scrollforward"
            android:layout_alignTop="@+id/text_scrollforward"
            android:background="#c0ffffff" />

        <TextView
            android:id="@+id/text_pages"
            android:layout_width="match_parent"
            android:layout_height="wrap_content"
            android:layout_alignBaseline="@+id/text_scrollforward"
            android:layout_toLeftOf="@+id/text_scrollforward"
            android:layout_toRightOf="@+id/text_scrollback"
            android:gravity="center"
            android:textColor="@color/mediumgrey"
            android:textSize="17sp" />
    </RelativeLayout>

</RelativeLayout>