diff --git a/rest_framework/serializers.py b/rest_framework/serializers.py
index 99d36a8a5..70ec08a20 100644
--- a/rest_framework/serializers.py
+++ b/rest_framework/serializers.py
@@ -1131,9 +1131,14 @@ class ModelSerializer(Serializer):
         field_kwargs = get_relation_kwargs(field_name, relation_info)
 
         to_field = field_kwargs.pop('to_field', None)
-        if to_field and not relation_info.related_model._meta.get_field(to_field).primary_key:
-            field_kwargs['slug_field'] = to_field
-            field_class = self.serializer_related_to_field
+        if relation_info.reverse:
+            if to_field and not relation_info.related_model_field.related_fields[0][1].primary_key:
+                field_kwargs['slug_field'] = to_field
+                field_class = self.serializer_related_to_field
+        else:
+            if to_field and not relation_info.related_model._meta.get_field(to_field).primary_key:
+                field_kwargs['slug_field'] = to_field
+                field_class = self.serializer_related_to_field
 
         # `view_name` is only valid for hyperlinked relationships.
         if not issubclass(field_class, HyperlinkedRelatedField):
diff --git a/rest_framework/utils/field_mapping.py b/rest_framework/utils/field_mapping.py
index 1d8cb22f2..8642ccf55 100644
--- a/rest_framework/utils/field_mapping.py
+++ b/rest_framework/utils/field_mapping.py
@@ -238,7 +238,7 @@ def get_relation_kwargs(field_name, relation_info):
     """
     Creates a default instance of a flat relational field.
     """
-    model_field, related_model, to_many, to_field, has_through_model = relation_info
+    model_field, related_model, related_model_field, to_many, to_field, has_through_model, reverse = relation_info
     kwargs = {
         'queryset': related_model._default_manager,
         'view_name': get_detail_view_name(related_model)
diff --git a/rest_framework/utils/model_meta.py b/rest_framework/utils/model_meta.py
index f151c6f36..c496d2711 100644
--- a/rest_framework/utils/model_meta.py
+++ b/rest_framework/utils/model_meta.py
@@ -29,9 +29,11 @@ FieldInfo = namedtuple('FieldResult', [
 RelationInfo = namedtuple('RelationInfo', [
     'model_field',
     'related_model',
+    'related_model_field',
     'to_many',
     'to_field',
-    'has_through_model'
+    'has_through_model',
+    'reverse'
 ])
 
 
@@ -108,9 +110,11 @@ def _get_forward_relationships(opts):
         forward_relations[field.name] = RelationInfo(
             model_field=field,
             related_model=_resolve_model(field.rel.to),
+            related_model_field=None,
             to_many=False,
             to_field=_get_to_field(field),
-            has_through_model=False
+            has_through_model=False,
+            reverse=False
         )
 
     # Deal with forward many-to-many relationships.
@@ -118,12 +122,14 @@ def _get_forward_relationships(opts):
         forward_relations[field.name] = RelationInfo(
             model_field=field,
             related_model=_resolve_model(field.rel.to),
+            related_model_field=None,
             to_many=True,
             # manytomany do not have to_fields
             to_field=None,
             has_through_model=(
                 not field.rel.through._meta.auto_created
-            )
+            ),
+            reverse=False
         )
 
     return forward_relations
@@ -144,9 +150,11 @@ def _get_reverse_relationships(opts):
         reverse_relations[accessor_name] = RelationInfo(
             model_field=None,
             related_model=related,
+            related_model_field=relation.field,
             to_many=relation.field.rel.multiple,
             to_field=_get_to_field(relation.field),
-            has_through_model=False
+            has_through_model=False,
+            reverse=True
         )
 
     # Deal with reverse many-to-many relationships.
@@ -156,13 +164,15 @@ def _get_reverse_relationships(opts):
         reverse_relations[accessor_name] = RelationInfo(
             model_field=None,
             related_model=related,
+            related_model_field=relation.field,
             to_many=True,
             # manytomany do not have to_fields
             to_field=None,
             has_through_model=(
                 (getattr(relation.field.rel, 'through', None) is not None) and
                 not relation.field.rel.through._meta.auto_created
-            )
+            ),
+            reverse=True
         )
 
     return reverse_relations
diff --git a/tests/test_model_serializer.py b/tests/test_model_serializer.py
index 57e540e7a..9154a28bb 100644
--- a/tests/test_model_serializer.py
+++ b/tests/test_model_serializer.py
@@ -899,3 +899,38 @@ class TestDecimalFieldMappings(TestCase):
         serializer = TestSerializer()
 
         assert len(serializer.fields['decimal_field'].validators) == 2
+
+
+class Issue3674Test(TestCase):
+    def test_nonPK_foreignkey_model_serializer(self):
+        class TestParentModel(models.Model):
+            title = models.CharField(max_length=64)
+
+        class TestChildModel(models.Model):
+            parent = models.ForeignKey(TestParentModel, related_name='children')
+            value = models.CharField(primary_key=True, max_length=64)
+
+        class TestChildModelSerializer(serializers.ModelSerializer):
+            class Meta:
+                model = TestChildModel
+                fields = ('value', 'parent')
+
+        class TestParentModelSerializer(serializers.ModelSerializer):
+            class Meta:
+                model = TestParentModel
+                fields = ('id', 'title', 'children')
+
+        parent_expected = dedent("""
+            TestParentModelSerializer():
+                id = IntegerField(label='ID', read_only=True)
+                title = CharField(max_length=64)
+                children = PrimaryKeyRelatedField(many=True, queryset=TestChildModel.objects.all())
+        """)
+        self.assertEqual(unicode_repr(TestParentModelSerializer()), parent_expected)
+
+        child_expected = dedent("""
+            TestChildModelSerializer():
+                value = CharField(max_length=64, validators=[<UniqueValidator(queryset=TestChildModel.objects.all())>])
+                parent = PrimaryKeyRelatedField(queryset=TestParentModel.objects.all())
+        """)
+        self.assertEqual(unicode_repr(TestChildModelSerializer()), child_expected)