diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py
index 35fb8fcf0..99d36a8a5 100644
--- a/rest_framework/serializers.py
+++ b/rest_framework/serializers.py
@@ -1131,7 +1131,7 @@ class ModelSerializer(Serializer):
         field_kwargs = get_relation_kwargs(field_name, relation_info)
 
         to_field = field_kwargs.pop('to_field', None)
-        if to_field and to_field != 'id':
+        if to_field and not relation_info.related_model._meta.get_field(to_field).primary_key:
             field_kwargs['slug_field'] = to_field
             field_class = self.serializer_related_to_field