diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py
index 99d36a8a5..8ed812099 100644
--- a/rest_framework/serializers.py
+++ b/rest_framework/serializers.py
@@ -1131,9 +1131,14 @@ class ModelSerializer(Serializer):
         field_kwargs = get_relation_kwargs(field_name, relation_info)
 
         to_field = field_kwargs.pop('to_field', None)
-        if to_field and not relation_info.related_model._meta.get_field(to_field).primary_key:
-            field_kwargs['slug_field'] = to_field
-            field_class = self.serializer_related_to_field
+        if relation_info.reverse:
+            if to_field and not relation_info.related_model_field.related_field.primary_key:
+                field_kwargs['slug_field'] = to_field
+                field_class = self.serializer_related_to_field
+        else:
+            if to_field and not relation_info.related_model._meta.get_field(to_field).primary_key:
+                field_kwargs['slug_field'] = to_field
+                field_class = self.serializer_related_to_field
 
         # `view_name` is only valid for hyperlinked relationships.
         if not issubclass(field_class, HyperlinkedRelatedField):
diff --git a/rest_framework/utils/field_mapping.py b/rest_framework/utils/field_mapping.py
index 1d8cb22f2..8642ccf55 100644
--- a/rest_framework/utils/field_mapping.py
+++ b/rest_framework/utils/field_mapping.py
@@ -238,7 +238,7 @@ def get_relation_kwargs(field_name, relation_info):
     """
     Creates a default instance of a flat relational field.
     """
-    model_field, related_model, to_many, to_field, has_through_model = relation_info
+    model_field, related_model, related_model_field, to_many, to_field, has_through_model, reverse = relation_info
     kwargs = {
         'queryset': related_model._default_manager,
         'view_name': get_detail_view_name(related_model)
diff --git a/rest_framework/utils/model_meta.py b/rest_framework/utils/model_meta.py
index c2afd9691..c496d2711 100644
--- a/rest_framework/utils/model_meta.py
+++ b/rest_framework/utils/model_meta.py
@@ -29,9 +29,11 @@ FieldInfo = namedtuple('FieldResult', [
 RelationInfo = namedtuple('RelationInfo', [
     'model_field',
     'related_model',
+    'related_model_field',
     'to_many',
     'to_field',
-    'has_through_model'
+    'has_through_model',
+    'reverse'
 ])
 
 
@@ -108,9 +110,11 @@ def _get_forward_relationships(opts):
         forward_relations[field.name] = RelationInfo(
             model_field=field,
             related_model=_resolve_model(field.rel.to),
+            related_model_field=None,
             to_many=False,
             to_field=_get_to_field(field),
-            has_through_model=False
+            has_through_model=False,
+            reverse=False
         )
 
     # Deal with forward many-to-many relationships.
@@ -118,12 +122,14 @@ def _get_forward_relationships(opts):
         forward_relations[field.name] = RelationInfo(
             model_field=field,
             related_model=_resolve_model(field.rel.to),
+            related_model_field=None,
             to_many=True,
             # manytomany do not have to_fields
             to_field=None,
             has_through_model=(
                 not field.rel.through._meta.auto_created
-            )
+            ),
+            reverse=False
         )
 
     return forward_relations
@@ -144,9 +150,11 @@ def _get_reverse_relationships(opts):
         reverse_relations[accessor_name] = RelationInfo(
             model_field=None,
             related_model=related,
+            related_model_field=relation.field,
             to_many=relation.field.rel.multiple,
-            to_field=_get_to_field(relation.field.model._meta.pk),
-            has_through_model=False
+            to_field=_get_to_field(relation.field),
+            has_through_model=False,
+            reverse=True
         )
 
     # Deal with reverse many-to-many relationships.
@@ -156,13 +164,15 @@ def _get_reverse_relationships(opts):
         reverse_relations[accessor_name] = RelationInfo(
             model_field=None,
             related_model=related,
+            related_model_field=relation.field,
             to_many=True,
             # manytomany do not have to_fields
             to_field=None,
             has_through_model=(
                 (getattr(relation.field.rel, 'through', None) is not None) and
                 not relation.field.rel.through._meta.auto_created
-            )
+            ),
+            reverse=True
         )
 
     return reverse_relations